diff --git a/.github/workflows/build-dependency-cache.yml b/.github/workflows/build-dependency-cache.yml deleted file mode 100644 index b1498d894f..0000000000 --- a/.github/workflows/build-dependency-cache.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -# Creates a cache of the env directory -# Documentation for the syntax of this file is located -# https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions - -# The workflow name will show up in the action tab on github during execution -# https://github.com/VOLTTRON/volttron/actions (or if you are pushing to your own fork change the user) - -name: cache virtual env - -# Check the cache when a push happens to the repository. -on: [push] - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] - python-version: [3.6, 3.7, 3.8] - steps: - # checkout the volttron repository and set current direectory to it - - uses: actions/checkout@v2 - - - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - id: envcache - uses: actions/cache@v2 - env: - cache-name: cache-env - with: - # This path is specific to Ubuntu - path: ./env - # Look to see if there is a cache hit for the corresponding requirements file - key: env-${{ matrix.os }}-${{matrix.python-version}}-${{hashFiles('requirements.py')}} - #-${{hashFiles('requirements.py')}} - # env-${{ matrix.os }}-${{matrix.python-version}} - - restore-keys: | - env-${{ matrix.os }}-${{matrix.python-version}}-${{hashFiles('requirements.py')}} - env-${{ matrix.os }}-${{matrix.python-version}} - - - name: Check env existance - id: check_files - uses: andstor/file-existence-action@v1 - with: - files: "env/bin/activate" - - - name: Install dependencies - if: steps.check_files.outputs.files_exists != 'true' - run: | - pip install wheel - python bootstrap.py --all --force - -# Only works on default branch of the target repo -# - name: Repository Dispatch -# uses: peter-evans/repository-dispatch@v1 -# with: -# token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }} -# repository: ${{ github.repository }} -# event-type: my-event -# client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' diff --git a/.github/workflows/pytest-auth.yml b/.github/workflows/pytest-auth.yml index 04f68b40ff..3a198cd8f7 100644 --- a/.github/workflows/pytest-auth.yml +++ b/.github/workflows/pytest-auth.yml @@ -38,6 +38,12 @@ jobs: # checkout the volttron repository and set current direectory to it - uses: actions/checkout@v2 + # Install erlang for rabbitmq + - name: Install erlang + if: matrix.os != 'ubuntu-20.04' + run: | + sudo scripts/rabbit_dependencies.sh debian ${{ matrix.os }} + # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -53,11 +59,17 @@ jobs: files: "env/bin/activate" # This step is only run if the cache wasn't able to be restored. - - name: Install dependencies - if: steps.check_files.outputs.files_exists != 'true' + - name: Install dependencies including rmq + if: steps.check_files.outputs.files_exists != 'true' && matrix.os != 'ubuntu-20.04' + run: | + pip install wheel + python bootstrap.py --all --rabbitmq --force + + - name: Install dependencies other than rmq + if: steps.check_files.outputs.files_exists != 'true' && matrix.os == 'ubuntu-20.04' run: | pip install wheel - python bootstrap.py --all --force + python bootstrap.py --all --force - name: Install volttron run: | diff --git a/.github/workflows/pytest-testutils.yml b/.github/workflows/pytest-testutils.yml index cd915fdeb6..88472e1309 100644 --- a/.github/workflows/pytest-testutils.yml +++ b/.github/workflows/pytest-testutils.yml @@ -27,6 +27,12 @@ jobs: # checkout the volttron repository and set current direectory to it - uses: actions/checkout@v2 + # Install erlang for rabbitmq + - name: Install erlang + if: matrix.os != 'ubuntu-20.04' + run: | + sudo scripts/rabbit_dependencies.sh debian ${{ matrix.os }} + # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} @@ -41,12 +47,18 @@ jobs: with: files: "env/bin/activate" - # if cache wasn't restored then do an installation of the dependencies - - name: Install dependencies - if: steps.check_files.outputs.files_exists != 'true' + # This step is only run if the cache wasn't able to be restored. + - name: Install dependencies including rmq + if: steps.check_files.outputs.files_exists != 'true' && matrix.os != 'ubuntu-20.04' + run: | + pip install wheel + python bootstrap.py --all --rabbitmq --force + + - name: Install dependencies other than rmq + if: steps.check_files.outputs.files_exists != 'true' && matrix.os == 'ubuntu-20.04' run: | pip install wheel - python bootstrap.py --all --force + python bootstrap.py --all --force - name: Install volttron run: | diff --git a/services/ops/FailoverAgent/README.md b/deprecated/FailoverAgent/README.md similarity index 100% rename from services/ops/FailoverAgent/README.md rename to deprecated/FailoverAgent/README.md diff --git a/services/ops/FailoverAgent/config b/deprecated/FailoverAgent/config similarity index 100% rename from services/ops/FailoverAgent/config rename to deprecated/FailoverAgent/config diff --git a/services/ops/FailoverAgent/conftest.py b/deprecated/FailoverAgent/conftest.py similarity index 100% rename from services/ops/FailoverAgent/conftest.py rename to deprecated/FailoverAgent/conftest.py diff --git a/services/core/MarketServiceAgent/market_service/__init__.py b/deprecated/FailoverAgent/failover/__init__.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/__init__.py rename to deprecated/FailoverAgent/failover/__init__.py diff --git a/services/ops/FailoverAgent/failover/agent.py b/deprecated/FailoverAgent/failover/agent.py similarity index 100% rename from services/ops/FailoverAgent/failover/agent.py rename to deprecated/FailoverAgent/failover/agent.py diff --git a/services/core/OpenADRVenAgent/setup.py b/deprecated/FailoverAgent/setup.py similarity index 100% rename from services/core/OpenADRVenAgent/setup.py rename to deprecated/FailoverAgent/setup.py diff --git a/services/ops/FailoverAgent/tests/test_failover.py b/deprecated/FailoverAgent/tests/test_failover.py similarity index 100% rename from services/ops/FailoverAgent/tests/test_failover.py rename to deprecated/FailoverAgent/tests/test_failover.py diff --git a/services/ops/FailoverAgent/tests/test_simple_failover.py b/deprecated/FailoverAgent/tests/test_simple_failover.py similarity index 100% rename from services/ops/FailoverAgent/tests/test_simple_failover.py rename to deprecated/FailoverAgent/tests/test_simple_failover.py diff --git a/docs/source/driver-framework/drivers-overview.rst b/docs/source/driver-framework/drivers-overview.rst index 074230e4ac..9ef2b5f240 100644 --- a/docs/source/driver-framework/drivers-overview.rst +++ b/docs/source/driver-framework/drivers-overview.rst @@ -10,8 +10,8 @@ Platform Driver process. Driver instances are created by the Platform Driver when a new driver configuration is added to the configuration store. Drivers use the following topic pattern `devices///`. When a configuration file is added -to the Platform Driver's store using this pattern, the Platform Driver creates a Driver Agent. The Driver agent is in turn -instantiated with a instance of the Interface class corresponding to the `driver_type` parameter in the configuration +to the Platform Driver's store using this pattern, the Platform Driver creates a Driver Agent. The Driver agent is then +instantiated with an instance of the Interface class corresponding to the `driver_type` parameter in the configuration file. The Interface class is responsible for implementing the communication paradigms of a device or protocol. Once configured, the Platform Driver periodically polls the Driver Agent for data which is collected from the interface class. Additionally, points can be requested ad-hoc via the Platform Driver's JSON-RPC method "get_point". Points may be set @@ -21,12 +21,10 @@ by using JSON-RPC with the Actuator agent to set up a schedule and calling the " Driver Conventions ****************** -- Drivers are polled by the Platform Driver agent and values can be set using the `Actuator Agent` -- Drivers should have a 1-to-1 relationship with a device -- Driver modules should be written in Python files in the `services/core/PlatformDriverAgent/platform_driver/interfaces` - directory in the VOLTTRON repository. The platform driver will search for a Python file in this directory matching the - name provided by the `driver_type` value from the driver configuration when creating the Driver agent. -- Driver code consists of an Interface class (exactly named), supported in most cases by one or more Register classes +* Drivers are polled by the Platform Driver agent and values can be set using the `Actuator Agent`. +* Drivers should have a 1-to-1 relationship with a device. +* Driver modules should be written in Python files in the `services/core/PlatformDriverAgent/platform_driver/interfaces` directory in the VOLTTRON repository. The platform driver will search for a Python file in this directory matching the name provided by the `driver_type` value from the driver configuration when creating the Driver agent. +* Driver code consists of an Interface class (exactly named), supported in most cases by one or more Register classes. .. _Driver_Communication: @@ -51,19 +49,19 @@ The diagram features several entities that comprise the platform and its connect message bus is built around existing message bus software; currently VOLTTRON supports RabbitMQ and ZeroMQ. The VOLTTRON integration includes Pub/Sub and JSON RPC interfaces for agent and driver communication. * VOLTTRON Platform Agents and Subsystems - These agents and subsystems are installed on the platform to manage the - platform. They provide many user facing functions, aid in communication and manage other agents and drivers. -* User's Agents - These agents are either agents included in the core repository but installed by a user, or user built + platform. They provide many user facing functions, aid in communication, and manage other agents and drivers. +* User's Agents - These agents are either agents included in the core repository but installed by a user or built by an end-user's agent modules. They may perform a huge variety of user specified tasks, including data collection, device control, simulation, etc. -* Platform Driver Agent - This agent is installed by a user to facilitate communication with drivers. Drivers should not - communicated with directly - the platform driver implements several features for communicating with drivers to ensure +* Platform Driver Agent - This agent facilitates communication with drivers. Agents should not + communicate directly with drivers. The platform driver implements several features for communicating with drivers to ensure smooth operation and consistent driver behavior. -* Actuator agent - This agent is installed by user to provide scheduling capability for controlling drivers. The +* Actuator agent - This agent provides scheduling capability for controlling drivers. The Platform Driver does not include protections for race conditions, etc. It is always recommended to use the Actuator agent to set values on a device. * Device Driver - Drivers are special purpose agents which provide an interface between the platform driver and devices - such as Modbus, and BACnet devices. Drivers implement a specific set of features for protecting device communication - ensuring uniform behaviors across different devices. + such as Modbus and BACnet devices. Drivers implement a specific set of features for protecting device communication and + ensure uniform behaviors across different devices. * Device - Devices may be low level physical computers for controlling various systems such as PLCs (Programmable Logic Controller), devices which communicate on the local network (such as a Smart T.V.), or devices which are accessed via a remote web API (other smart devices). @@ -76,9 +74,9 @@ Connectivity of the platform follows the following paradigm: * Platform agents (including the Platform Driver and Actuator), subsystems, and user agents communicate with the message bus via a publish/subscribe system. -* Agents can communicate "directly" to each other via JSONRPC calls - JSONRPC calls use the VOLTTRON message bus router +* Agents can communicate "directly" to each other via JSONRPC (RPC). A JSONRPC call uses the VOLTTRON message bus router to "direct" messages to an intended recipient. RPC calls from an agent specify a function for the recipient to - perform including input parameters, and the response to the sender should contain the value output by the specified + perform including input parameters; the response to the sender should contain the value output by the specified function. * The Platform Driver will periodically poll device drivers. This functionality is intentionally not user-facing. The Platform Driver iterates over the configured drivers and calls their respective "scrape_all" methods. This will trigger @@ -151,7 +149,7 @@ Installing the Fake Driver ************************** The Fake Driver is included as a way to quickly see data published to the message bus in a format that mimics what a -true driver would produce. This is a simple implementation of the VOLTTRON driver framework. +real driver would produce. This is a simple implementation of the VOLTTRON driver framework. See :ref:`instructions for installing the fake driver ` diff --git a/docs/source/driver-framework/platform-driver/platform-driver.rst b/docs/source/driver-framework/platform-driver/platform-driver.rst index 716e2b3d1e..9a40b20854 100644 --- a/docs/source/driver-framework/platform-driver/platform-driver.rst +++ b/docs/source/driver-framework/platform-driver/platform-driver.rst @@ -88,7 +88,7 @@ The easiest way to install the requirements for drivers included in the VOLTTRON Platform Driver Configuration ============================= -The Platform Driver Agent configuration consists of general settings for all devices. The default values of the Master +The Platform Driver Agent configuration consists of general settings for all devices. The default values of the Platform Driver should be sufficient for most users. The user may optionally change the interval between device scrapes with the driver_scrape_interval. @@ -152,13 +152,13 @@ The following settings are required for all device configurations: - **driver_config** - Driver specific setting go here. See below for driver specific settings. - **driver_type** - Type of driver to use for this device: bacnet, modbus, fake, etc. - **registry_config** - Reference to a configuration file in the configuration store for registers - on the device. See the `Registry-Configuration-File`_ section below or - and the :ref:`Adding Device Configurations to the Configuration Store ` section in + on the device. See the `Registry-Configuration-File`_ section below and/or + the :ref:`Adding Device Configurations to the Configuration Store ` section in the driver framework docs. These settings are optional: - - **interval** - Period which to scrape the device and publish the results in seconds. Defaults to 60 seconds. + - **interval** - Period to scrape the device and publish the results in seconds. Defaults to 60 seconds. - **heart_beat_point** - A Point which to toggle to indicate a heartbeat to the device. A point with this ``Volttron Point Name`` must exist in the registry. If this setting is missing the driver will not send a heart beat signal to the device. Heart beats are triggered by the :ref:`Actuator Agent ` which must be running to @@ -167,7 +167,7 @@ These settings are optional: These settings are used to create the topic that this device will be referenced by following the VOLTTRON convention of ``{campus}/{building}/{unit}``. This will also be the topic published on, when the device is periodically scraped for -it's current state. +its current state. The topic used to reference the device is derived from the name of the device configuration in the store. See the :ref:`Adding Device Configurations to the Configuration Store ` section of the driver @@ -239,7 +239,7 @@ is accessible with the Actuator Agent via `PNNL/ISB1/vav1`. The name of a registry configuration must match the name used to refer to it in the driver configuration. The reference is not case sensitive. -If the Platform Driver Agent is running any changes to the configuration store will immediately affect the running devices +If the Platform Driver Agent is running, any changes to the configuration store will immediately affect the running devices according to the changes. Example @@ -279,7 +279,7 @@ Converting Old Style Configuration The new Platform Driver no longer supports the old style of device configuration. The old `device_list` setting is ignored. -To simplify updating to the new format `scripts/update_platform_driver_config.py` is provide to automatically update to +To simplify updating to the new format, `scripts/update_platform_driver_config.py` is provided to automatically update to the new configuration format. With the platform running run: @@ -288,12 +288,12 @@ With the platform running run: python scripts/update_platform_driver_config.py -old_configuration`` is the main configuration file in the old format. The script automatically modifies the driver +`old_configuration` is the main configuration file in the old format. The script automatically modifies the driver files to create references to CSV files and adds the CSV files with the appropriate name. `output` is the target output directory. -If the ``--keep-old`` switch is used the old configurations in the output directory (if any) will not be deleted before +If the ``--keep-old`` switch is used, the old configurations in the output directory (if any) will not be deleted before new configurations are created. Matching names will still be overwritten. The output from `scripts/update_platform_driver_config.py` can be automatically added to the configuration store @@ -307,7 +307,7 @@ the process of changing and updating a large number of configurations. See the ` Device Scalability Settings --------------------------- -In order to improve the scalability of the platform unneeded device state publishes for a device can be turned off. +To improve the scalability of the platform, unneeded device state publishes for a device can be turned off. All of the following setting are optional and will override the value set in the main platform driver configuration. - **publish_depth_first_all** - Enable "depth first" publish of all points to a single topic. @@ -339,8 +339,8 @@ Polling Once running, the Platform Driver will spawn drivers using the `driver_type` parameter of the :ref:`driver configuration ` and periodically poll devices for all point data specified in -the :ref:`registry configuration ` (at the interval specified by the interval parameter -of the driver configuration). +the :ref:`registry configuration ` at the interval specified by the interval parameter +of the driver configuration. By default, the value of each register on a device is published 4 different ways when the device state is published. Consider the following settings in a driver configuration stored under the name ``devices/pnnl/isb1/vav1``: @@ -355,14 +355,14 @@ Consider the following settings in a driver configuration stored under the name "registry_config":"config://registry_configs/vav.csv", } -In the `vav.csv` file is a register with the name `temperature`. For these examples the current value of the +In the `vav.csv` file, a register has the name `temperature`. For these examples the current value of the register on the device happens to be 75.2 and the meta data is .. code-block:: json {"units": "F"} -When the driver publishes the device state the following 2 things will be published for this register: +When the driver publishes the device state the following two things will be published for this register: A "depth first" publish to the topic `devices/pnnl/isb1/vav1/temperature` with the following message: diff --git a/examples/SchedulerExample/schedule_example/agent.py b/examples/SchedulerExample/schedule_example/agent.py index 6ae69a9250..3371418261 100644 --- a/examples/SchedulerExample/schedule_example/agent.py +++ b/examples/SchedulerExample/schedule_example/agent.py @@ -72,7 +72,7 @@ def schedule_example(config_path, **kwargs): class SchedulerExample(Agent): '''This agent can be used to demonstrate scheduling and - acutation of devices. It reserves a non-existant device, then + acutation of devices. It reserves a non-existent device, then acts when its time comes up. Since there is no device, this will cause an error. ''' @@ -126,18 +126,16 @@ def publish_schedule(self): msg = [ - ['campus/building/unit',start,end] - #Could add more devices - # ["campus/building/device1", #First time slot. - # "2014-1-31 12:27:00", #Start of time slot. - # "2016-1-31 12:29:00"], #End of time slot. - # ["campus/building/device2", #Second time slot. - # "2014-1-31 12:26:00", #Start of time slot. - # "2016-1-31 12:30:00"], #End of time slot. - # ["campus/building/device3", #Third time slot. - # "2014-1-31 12:30:00", #Start of time slot. - # "2016-1-31 12:32:00"], #End of time slot. - #etc... + ['campus/building/unit', start, end], + ["campus/building/device1", #First time slot. + "2014-1-31 12:27:00", #Start of time slot. + "2016-1-31 12:29:00"], #End of time slot. + ["campus/building/device2", #Second time slot. + "2014-1-31 12:26:00", #Start of time slot. + "2016-1-31 12:30:00"], #End of time slot. + ["campus/building/device3", #Third time slot. + "2014-1-31 12:30:00", #Start of time slot. + "2016-1-31 12:32:00"], #End of time slot. ] self.vip.pubsub.publish( 'pubsub', topics.ACTUATOR_SCHEDULE_REQUEST, headers, msg) @@ -160,7 +158,7 @@ def use_rpc(self): msg).get(timeout=10) print("schedule result", result) except Exception as e: - print ("Could not contact actuator. Is it running?") + print("Could not contact actuator. Is it running?") print(e) return @@ -173,13 +171,22 @@ def use_rpc(self): 'campus/building/unit3/some_point', '0.0').get(timeout=10) print("Set result", result) + + topic_values = [] + for x in msg: + topic = x[0] + value = '0.0' + topic_values.append((topic, value)) + result = self.vip.rpc.call( + 'platform.actuator', + 'set_multiple_points', + self.core.identity, + topic_values).get(timeout=10) + print("Set_multiple_points result", result) except Exception as e: - print ("Expected to fail since there is no real device to set") + print("Expected to fail since there is no real device to set") print(e) - - - - + Agent.__name__ = 'ScheduleExampleAgent' return SchedulerExample(**kwargs) diff --git a/examples/defaultLogConfig.ini b/examples/defaultLogConfig.ini deleted file mode 100644 index 8d6f9aec8f..0000000000 --- a/examples/defaultLogConfig.ini +++ /dev/null @@ -1,30 +0,0 @@ -[loggers] -keys=root,debug01 - -[handlers] -keys=h_rotating - -[formatters] -keys=f_rotating - -[logger_root] -level=NOTSET -handlers=h_rotating - -[logger_debug01] -level=DEBUG -handlers=h_rotationg -propagate=1 -qualname=debug.log - -[handler_h_rotating] -class=TimedRotatingFileHandler -level=DEBUG -formatter=f_rotating -args=(volttron.log,when='midnight',backupCount=7) - -# Looks like the formatter may get overwritten by the code -[formatter_f_rotating] -format= -datefmt= -class=logging.Formatter diff --git a/pytest.ini b/pytest.ini index 1048740d1a..27313bda08 100644 --- a/pytest.ini +++ b/pytest.ini @@ -64,3 +64,4 @@ markers = influxdbutils: level one integration tests for influxdb federation: Tests for rabbitmq federation communication shovel: Tests for rabbitmq shovel communication + contrib: tests for community-contributed agents diff --git a/scripts/rabbit_dependencies.sh b/scripts/rabbit_dependencies.sh index 28c7326d5c..ab1f7f542e 100755 --- a/scripts/rabbit_dependencies.sh +++ b/scripts/rabbit_dependencies.sh @@ -2,6 +2,8 @@ set -e list=( bionic buster ) +declare -A ubuntu_versions +ubuntu_versions=( ["ubuntu-16.04"]="xenial" ["ubuntu-18.04"]="bionic" ["ubuntu-20.04"]="focal") function exit_on_error { rc=$? @@ -16,8 +18,8 @@ function exit_on_error { function print_usage { echo " Command Usage: -/rabbit_dependencies.sh -Valid Raspbian/Debian distributions: ${list[@]} +/rabbit_dependencies.sh or centos version> +Valid Raspbian/Debian distributions: ${list[@]} ${!ubuntu_versions[@]} Valid centos versions: 6, 7, 8 " exit 0 @@ -72,6 +74,17 @@ function install_on_debian { fi done + if [[ "$FOUND" != "1" ]]; then + # check if ubuntu-version was passed if so map it to name + for ubuntu_version in "${!ubuntu_versions[@]}"; do + if [[ "$DIST" == "$ubuntu_version" ]]; then + FOUND=1 + DIST="${ubuntu_versions[$ubuntu_version]}" + break + fi + done + fi + if [[ "$FOUND" != "1" ]]; then echo "Invalid distribution found" print_usage @@ -79,7 +92,11 @@ function install_on_debian { echo "installing ERLANG" ${prefix} apt-get update - ${prefix} apt-get install -y apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + if [[ "$DIST" == "xenial" ]] || [[ "$DIST" == "bionic" ]]; then + ${prefix} apt-get install -y apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + else + ${prefix} apt-get install -y apt-transport-https libwxbase3.0-0v5 libwxgtk3.0-gtk3-0v5 libsctp1 build-essential python-dev openssl libssl-dev libevent-dev git + fi set +e ${prefix} apt-get purge -yf erlang* set -e diff --git a/services/core/MarketServiceAgent/IDENTITY b/services/contrib/MarketServiceAgent/IDENTITY similarity index 100% rename from services/core/MarketServiceAgent/IDENTITY rename to services/contrib/MarketServiceAgent/IDENTITY diff --git a/services/core/MarketServiceAgent/README.md b/services/contrib/MarketServiceAgent/README.md similarity index 100% rename from services/core/MarketServiceAgent/README.md rename to services/contrib/MarketServiceAgent/README.md diff --git a/services/core/MarketServiceAgent/config b/services/contrib/MarketServiceAgent/config similarity index 100% rename from services/core/MarketServiceAgent/config rename to services/contrib/MarketServiceAgent/config diff --git a/services/core/MarketServiceAgent/conftest.py b/services/contrib/MarketServiceAgent/conftest.py similarity index 100% rename from services/core/MarketServiceAgent/conftest.py rename to services/contrib/MarketServiceAgent/conftest.py diff --git a/services/core/MongodbHistorian/mongodb/__init__.py b/services/contrib/MarketServiceAgent/market_service/__init__.py similarity index 100% rename from services/core/MongodbHistorian/mongodb/__init__.py rename to services/contrib/MarketServiceAgent/market_service/__init__.py diff --git a/services/core/MarketServiceAgent/market_service/agent.py b/services/contrib/MarketServiceAgent/market_service/agent.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/agent.py rename to services/contrib/MarketServiceAgent/market_service/agent.py diff --git a/services/core/MarketServiceAgent/market_service/director.py b/services/contrib/MarketServiceAgent/market_service/director.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/director.py rename to services/contrib/MarketServiceAgent/market_service/director.py diff --git a/services/core/MarketServiceAgent/market_service/market.py b/services/contrib/MarketServiceAgent/market_service/market.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/market.py rename to services/contrib/MarketServiceAgent/market_service/market.py diff --git a/services/core/MarketServiceAgent/market_service/market_list.py b/services/contrib/MarketServiceAgent/market_service/market_list.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/market_list.py rename to services/contrib/MarketServiceAgent/market_service/market_list.py diff --git a/services/core/MarketServiceAgent/market_service/market_participant.py b/services/contrib/MarketServiceAgent/market_service/market_participant.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/market_participant.py rename to services/contrib/MarketServiceAgent/market_service/market_participant.py diff --git a/services/core/MarketServiceAgent/market_service/market_state.py b/services/contrib/MarketServiceAgent/market_service/market_state.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/market_state.py rename to services/contrib/MarketServiceAgent/market_service/market_state.py diff --git a/services/core/MarketServiceAgent/market_service/offer_manager.py b/services/contrib/MarketServiceAgent/market_service/offer_manager.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/offer_manager.py rename to services/contrib/MarketServiceAgent/market_service/offer_manager.py diff --git a/services/core/MarketServiceAgent/market_service/reservation_manager.py b/services/contrib/MarketServiceAgent/market_service/reservation_manager.py similarity index 100% rename from services/core/MarketServiceAgent/market_service/reservation_manager.py rename to services/contrib/MarketServiceAgent/market_service/reservation_manager.py diff --git a/services/core/MarketServiceAgent/requirements.txt b/services/contrib/MarketServiceAgent/requirements.txt similarity index 100% rename from services/core/MarketServiceAgent/requirements.txt rename to services/contrib/MarketServiceAgent/requirements.txt diff --git a/services/core/MarketServiceAgent/setup.py b/services/contrib/MarketServiceAgent/setup.py similarity index 100% rename from services/core/MarketServiceAgent/setup.py rename to services/contrib/MarketServiceAgent/setup.py diff --git a/services/core/MarketServiceAgent/tests/test_market.py b/services/contrib/MarketServiceAgent/tests/test_market.py similarity index 100% rename from services/core/MarketServiceAgent/tests/test_market.py rename to services/contrib/MarketServiceAgent/tests/test_market.py diff --git a/services/core/MarketServiceAgent/tests/test_market_list.py b/services/contrib/MarketServiceAgent/tests/test_market_list.py similarity index 100% rename from services/core/MarketServiceAgent/tests/test_market_list.py rename to services/contrib/MarketServiceAgent/tests/test_market_list.py diff --git a/services/core/MarketServiceAgent/tests/test_market_service_agent.py b/services/contrib/MarketServiceAgent/tests/test_market_service_agent.py similarity index 100% rename from services/core/MarketServiceAgent/tests/test_market_service_agent.py rename to services/contrib/MarketServiceAgent/tests/test_market_service_agent.py diff --git a/services/core/MarketServiceAgent/tests/test_offer.py b/services/contrib/MarketServiceAgent/tests/test_offer.py similarity index 100% rename from services/core/MarketServiceAgent/tests/test_offer.py rename to services/contrib/MarketServiceAgent/tests/test_offer.py diff --git a/services/core/ActuatorAgent/README.md b/services/core/ActuatorAgent/README.md index 1b63be628b..75c4466274 100644 --- a/services/core/ActuatorAgent/README.md +++ b/services/core/ActuatorAgent/README.md @@ -5,6 +5,9 @@ devices. Agents may interact with the ActuatorAgent via either PUB/SUB or RPC, but it is recommended agents use RPC to interact with the ActuatorAgent. +For an example of an agent using RPC to interact with the ActuatorAgent, see the +`SchedulerExample agent `__ +from the Volttron repository. The PUB/SUB interface remains primarily for VOLTTRON 2.0 agents. diff --git a/services/core/ActuatorAgent/tests/actuator-no-lock.config b/services/core/ActuatorAgent/tests/actuator-no-lock.config index eade47ac61..6fe85e4a25 100644 --- a/services/core/ActuatorAgent/tests/actuator-no-lock.config +++ b/services/core/ActuatorAgent/tests/actuator-no-lock.config @@ -1,5 +1,5 @@ { "schedule_publish_interval": 10, "schedule_state_file": "actuator_state.test", - "allow_no_lock_write": true + "allow_no_lock_write": true # default is True } diff --git a/services/core/ActuatorAgent/tests/actuator_fixtures.py b/services/core/ActuatorAgent/tests/actuator_fixtures.py deleted file mode 100644 index c21834023e..0000000000 --- a/services/core/ActuatorAgent/tests/actuator_fixtures.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: -# -# Copyright 2020, Battelle Memorial Institute. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. -# -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 -# }}} - -import contextlib - -from mock import create_autospec - -from services.core.ActuatorAgent.actuator.agent import ActuatorAgent, ScheduleManager -from services.core.ActuatorAgent.actuator.scheduler import RequestResult - - -class MockedAsyncResult: - """ - This class is used to help mock Responses from the vip subsystem - """ - def __init__(self, result): - self.result = result - - def get(self): - return self.result - - -@contextlib.contextmanager -def get_actuator_agent(vip_identity: str = "fake_vip_identity", - vip_rpc_call_res: MockedAsyncResult = MockedAsyncResult("fake_result"), - vip_message_peer: str = None, - device_state: dict = {}, - slot_requests_res: RequestResult = RequestResult(True, {("agent-id-1", "task-id-1")}, ""), - cancel_schedule_result: RequestResult = None): - """ - Creates an Actuator agent and mocks all required dependencies for unit testing - :param vip_identity: the identity of the Agent's Subsystem - :param vip_rpc_call_res: the response returned when calling a method of the Agent's Subsystem - :param vip_message_peer: the identity of the Agent's VIP, which is used internally - :param device_state: a mapping between a path and a DeviceState; this is an protected field of the Agent - :param slot_requests_res: the response returned when calling request_slots method of Agent's Schedule Manager - :param cancel_schedule_result: the response retunred when callin cancel_task method of Agent's Schedule Manaager - :return: - """ - ActuatorAgent.core.identity = "fake_core_identity" - actuator_agent = ActuatorAgent() - if vip_identity is not None: - actuator_agent.driver_vip_identity = vip_identity - actuator_agent.vip.rpc.call.return_value = vip_rpc_call_res - actuator_agent.vip.rpc.context.vip_message.peer = vip_message_peer - actuator_agent._device_states = device_state - actuator_agent._schedule_manager = create_autospec(ScheduleManager) - actuator_agent._schedule_manager.request_slots.return_value = slot_requests_res - actuator_agent._schedule_manager.get_next_event_time.return_value = None - actuator_agent._schedule_manager.cancel_task.return_value = cancel_schedule_result - actuator_agent._schedule_manager.get_schedule_state.return_value = {} - actuator_agent.core.schedule.return_value = None - - try: - yield actuator_agent - finally: - actuator_agent.vip.reset_mock() diff --git a/services/core/ActuatorAgent/tests/test_actuator_rpc.py b/services/core/ActuatorAgent/tests/test_actuator_rpc.py index 245c49ceab..63d997c588 100644 --- a/services/core/ActuatorAgent/tests/test_actuator_rpc.py +++ b/services/core/ActuatorAgent/tests/test_actuator_rpc.py @@ -37,7 +37,7 @@ # }}} """ -Pytest test cases for testing actuator agent using rpc calls. +Pytest integration test cases for testing actuator agent using rpc calls. """ import json @@ -200,8 +200,14 @@ def cleanup(): return cleanup_parameters +@pytest.mark.parametrize("taskid, expected_result, expected_info", [ + ('task_schedule_success', SUCCESS, ''), + (1234, FAILURE, 'MALFORMED_REQUEST: TypeError: taskid must be a nonempty string'), + ('', FAILURE, 'MALFORMED_REQUEST: TypeError: taskid must be a nonempty string'), + (None, FAILURE, 'MISSING_TASK_ID') +]) @pytest.mark.actuator -def test_schedule_success(publish_agent, cancel_schedules): +def test_request_new_schedule(publish_agent, cancel_schedules, taskid, expected_result, expected_info): """ Test responses for successful schedule request :param publish_agent: fixture invoked to setup all agents necessary and @@ -212,7 +218,6 @@ def test_schedule_success(publish_agent, cancel_schedules): print ("\n**** test_schedule_success ****") # used by cancel_schedules agentid = TEST_AGENT - taskid = 'task_schedule_success' cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) start = str(datetime.now()) @@ -229,109 +234,25 @@ def test_schedule_success(publish_agent, cancel_schedules): msg).get(timeout=10) # expected result {'info': u'', 'data': {}, 'result': SUCCESS} print(result) - assert result['result'] == SUCCESS - - -@pytest.mark.actuator -def test_schedule_error_int_taskid(publish_agent): - """ - Test responses for successful schedule request with integer task id - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - """ - print("\n**** test_schedule_error_int_taskid ****") - agentid = TEST_AGENT - taskid = 1234 - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=1)) - msg = [ - ['fakedriver1', start, end] - ] - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == FAILURE - assert result['info'] == 'MALFORMED_REQUEST: TypeError: taskid must be a nonempty string' - - -@pytest.mark.actuator -def test_schedule_empty_taskid(publish_agent, cancel_schedules): - """ - Test responses for successful schedule request when task id is an empty - string - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param cancel_schedules: fixture used to cancel the schedule at the end - of test so that other tests can use the same device and time slot - """ - print("\n**** test_schedule_empty_taskid ****") - # used by cancel_schedules - agentid = TEST_AGENT - taskid = '' - cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=1)) - msg = [ - ['fakedriver1', start, end] - ] - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == FAILURE - assert result['info'] == 'MALFORMED_REQUEST: TypeError: taskid must be a nonempty string' - - -@pytest.mark.actuator -def test_schedule_error_none_taskid(publish_agent): - """ - Test error responses for schedule request with taskid = None - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - """ - print("\n**** test_schedule_error_none_taskid ****") - agentid = TEST_AGENT - taskid = None - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=1)) - msg = [ - ['fakedriver1', start, end] - ] - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - print(result) - assert result['result'] == FAILURE - assert result['info'] == 'MISSING_TASK_ID' + assert result['result'] == expected_result + if not result['info']: + assert result['info'] == expected_info +@pytest.mark.parametrize("invalid_priority, expected_info", [ + ('LOW2', 'INVALID_PRIORITY'), + (None, 'MISSING_PRIORITY') +]) @pytest.mark.actuator -def test_schedule_error_invalid_priority(publish_agent): +def test_request_new_schedule_should_return_failure_on_bad_priority(publish_agent, invalid_priority, expected_info): """ Test error responses for schedule request with an invalid priority :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing """ print("\n**** test_schedule_error_invalid_priority ****") - taskid = 'task_invalid_priority' + taskid = 'task_bad_priority' start = str(datetime.now()) end = str(datetime.now() + timedelta(seconds=1)) msg = [ @@ -343,16 +264,17 @@ def test_schedule_error_invalid_priority(publish_agent): REQUEST_NEW_SCHEDULE, TEST_AGENT, taskid, - 'LOW2', + # 'LOW2', + invalid_priority, msg).get(timeout=10) # expected result {'info': u'', 'data': {}, 'result': SUCCESS} print(result) assert result['result'] == FAILURE - assert result['info'] == 'INVALID_PRIORITY' + assert result['info'] == expected_info @pytest.mark.actuator -def test_schedule_error_empty_message(publish_agent): +def test_request_new_schedule_should_return_failure_on_empty_message(publish_agent): """ Test error responses for schedule request with an empty message :param publish_agent: fixture invoked to setup all agents necessary and @@ -361,9 +283,7 @@ def test_schedule_error_empty_message(publish_agent): print("\n**** test_schedule_error_empty_message ****") taskid = 'task_empty_message' - msg = [ - - ] + msg = [] result = publish_agent.vip.rpc.call( PLATFORM_ACTUATOR, REQUEST_NEW_SCHEDULE, @@ -378,7 +298,7 @@ def test_schedule_error_empty_message(publish_agent): @pytest.mark.actuator -def test_schedule_error_duplicate_task(publish_agent, cancel_schedules): +def test_request_new_schedule_should_return_failure_on_duplicate_taskid(publish_agent, cancel_schedules): """ Test error responses for schedule request with task id that is already in use @@ -407,6 +327,7 @@ def test_schedule_error_duplicate_task(publish_agent, cancel_schedules): PRIORITY_LOW, msg).get(timeout=10) assert result['result'] == SUCCESS + # new request with same task id result = publish_agent.vip.rpc.call( PLATFORM_ACTUATOR, @@ -422,36 +343,7 @@ def test_schedule_error_duplicate_task(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_schedule_error_none_priority(publish_agent): - """ - Test error responses for schedule request with priority = None - :param publish_agent: fixture invoked to setup all agents necessary - and returns an instance of Agent object used for publishing - """ - print("\n**** test_schedule_error_none_priority ****") - taskid = 'task_none_priority' - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=1)) - msg = [ - ['fakedriver0', start, end] - ] - - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - TEST_AGENT, - taskid, - None, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == FAILURE - assert result['info'] == 'MISSING_PRIORITY' - - -@pytest.mark.actuator -def test_schedule_error_malformed_request(publish_agent): +def test_reques_new_schedule_error_malformed_request(publish_agent): """ Test error responses for schedule request with malformed request - request with only a device name and start time and no stop time @@ -480,7 +372,7 @@ def test_schedule_error_malformed_request(publish_agent): @pytest.mark.actuator -def test_schedule_preempt_self(publish_agent, cancel_schedules): +def test_request_new_schedule_should_succeed_on_preempt_self(publish_agent, cancel_schedules): """ Test error response for schedule request through pubsub. Test schedule preemption by a higher priority task from the same agent. @@ -557,7 +449,7 @@ def test_schedule_preempt_self(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_schedule_preempt_active_task(publish_agent, cancel_schedules): +def test_request_new_schedule_should_suceed_on_preempt_active_task(publish_agent, cancel_schedules): """ Test error response for schedule request. Test schedule preemption of a actively running task with priority @@ -638,7 +530,7 @@ def test_schedule_preempt_active_task(publish_agent, cancel_schedules): # This test checks to see if a requestid is no longer valid. # Since request ids are always vip identities and only one agent # is scheduling devices the expected lock error is not raised. -def test_schedule_preempt_active_task_gracetime(publish_agent, cancel_schedules): +def test_request_new_schedule_preempt_active_task_gracetime(publish_agent, cancel_schedules): """ Test error response for schedule request. Test schedule preemption of a actively running task with priority LOW by @@ -656,7 +548,7 @@ def test_schedule_preempt_active_task_gracetime(publish_agent, cancel_schedules) agentid = 'new_agent' taskid = 'task_high_priority3' cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) - # add low prority task as well since it won't get cancelled till end of grace time + # add low priority task as well since it won't get cancelled till end of grace time cancel_schedules.append( {'agentid': TEST_AGENT, 'taskid': 'task_low_priority3'}) @@ -752,7 +644,7 @@ def test_schedule_preempt_active_task_gracetime(publish_agent, cancel_schedules) @pytest.mark.actuator -def test_schedule_preempt_error_active_task(publish_agent, cancel_schedules): +def test_request_new_schedule_should_return_failure_on_preempt_active_task(publish_agent, cancel_schedules): """ Test error response for schedule request. Test schedule preemption of a actively running task with priority LOW by @@ -810,7 +702,7 @@ def test_schedule_preempt_error_active_task(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_schedule_preempt_future_task(publish_agent, cancel_schedules): +def test_request_new_schedule_should_succeed_on_preempt_future_task(publish_agent, cancel_schedules): """ Test error response for schedule request. Test schedule preemption of a future task with priority LOW by a higher @@ -889,7 +781,7 @@ def test_schedule_preempt_future_task(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_schedule_conflict_self(publish_agent): +def test_request_new_schedule_should_return_failure_on_conflicting_time_slots(publish_agent): """ Test error response for schedule request. Test schedule with conflicting time slots in the same request @@ -921,16 +813,15 @@ def test_schedule_conflict_self(publish_agent): @pytest.mark.actuator -def test_schedule_conflict(publish_agent, cancel_schedules): +def test_request_new_schedule_should_return_failure_on_conflicting_schedules(publish_agent, cancel_schedules): """ - Test schedule conflict with existing schdeule + Test schedule conflict with existing schedule :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing :param cancel_schedules: fixture used to cancel the schedule at the end of test so that other tests can use the same device and time slot """ - print ("\n**** test_schedule_conflict ****") # set agentid and task id for cancel_schedules fixture agentid = TEST_AGENT taskid = 'task_conflict1' @@ -966,7 +857,7 @@ def test_schedule_conflict(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_schedule_overlap_success(publish_agent, cancel_schedules): +def test_request_new_schedule_should_succeed_on_overlap_time_slots(publish_agent, cancel_schedules): """ Test schedule where stop time of one requested time slot is the same as start time of another requested time slot. @@ -1004,29 +895,7 @@ def test_schedule_overlap_success(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_cancel_error_invalid_taskid(publish_agent): - """ - Test error responses for schedule request. Test invalid task id - - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance - of Agent object used for publishing - """ - print ("\n**** test_cancel_error_invalid_taskid ****") - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_CANCEL_SCHEDULE, - TEST_AGENT, - 'invalid_cancel', - ).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == FAILURE - assert result['info'] == 'TASK_ID_DOES_NOT_EXIST' - - -@pytest.mark.actuator -def test_cancel_success(publish_agent): +def test_request_cancel_schedule_should_succeed(publish_agent): """ Test successful schedule cancel @@ -1063,78 +932,67 @@ def test_cancel_success(publish_agent): @pytest.mark.actuator -def test_get_default(publish_agent): +def test_request_cancel_schedule_should_return_failure_on_invalid_taskid(publish_agent): """ - Test get default value of a point + Test error responses for schedule request. Test invalid task id :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing + returns an instance + of Agent object used for publishing """ - print ("\n**** test_get_default ****") - + print ("\n**** test_cancel_error_invalid_taskid ****") result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, # Target agent - 'get_point', # Method - 'fakedriver1/SampleWritableFloat1' # point + PLATFORM_ACTUATOR, + REQUEST_CANCEL_SCHEDULE, + TEST_AGENT, + 'invalid_cancel', ).get(timeout=10) + # expected result {'info': u'', 'data': {}, 'result': SUCCESS} print(result) - assert result == 10.0 + assert result['result'] == FAILURE + assert result['info'] == 'TASK_ID_DOES_NOT_EXIST' +# We need to test the getters first before proceeding to testing the other actuator methods because +# some methods mutate driver points AND all tests share the same publish_agent setup @pytest.mark.actuator -def test_get_success(publish_agent, cancel_schedules): +def test_get_point_should_succeed(publish_agent): """ - Test getting a float value of a point through pubsub - Expected Result - value of the point + Test get default value of a point :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing - :param cancel_schedules: fixture used to cancel the schedule at - the end of test so that other tests can use the same device and time slot """ - print("\n**** test_get_value_success ****") - agentid = TEST_AGENT - taskid = 'task_set_and_get' - cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=2)) - msg = [ - ['fakedriver1', start, end] - ] - - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == SUCCESS - - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver1/SampleWritableFloat1', # Point to set - 1.0 # New value - ).get(timeout=10) - assert result == 1.0 + print ("\n**** test_get_default ****") result = publish_agent.vip.rpc.call( PLATFORM_ACTUATOR, # Target agent 'get_point', # Method 'fakedriver1/SampleWritableFloat1' # point ).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} print(result) - assert result == 1.0 + assert result == 10.0 +@pytest.mark.parametrize("topics", [ + (['fakedriver0/SampleWritableFloat1', + 'fakedriver1/SampleWritableFloat1']), + ([['fakedriver0', 'SampleWritableFloat1'], + ['fakedriver1', 'SampleWritableFloat1']]) +]) @pytest.mark.actuator -def test_get_success_with_point(publish_agent, cancel_schedules): +def test_get_multiple_points_should_succeed(publish_agent, cancel_schedules, topics): + results, errors = publish_agent.vip.rpc.call( + 'platform.actuator', + 'get_multiple_points', + topics).get(timeout=10) + + assert results == {'fakedriver0/SampleWritableFloat1': 10.0, 'fakedriver1/SampleWritableFloat1': 10.0} + assert errors == {} + + +@pytest.mark.actuator +def test_set_point_then_get_point_should_succeed(publish_agent, cancel_schedules): """ Test getting a float value of a point through RPC Expected Result - value of the point @@ -1186,14 +1044,13 @@ def test_get_success_with_point(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_get_error_invalid_point(publish_agent): +def test_get_point_raises_remote_error_on_invalid_point(publish_agent): """ Test getting a float value of a point through RPC with invalid point :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing """ - print ("\n**** test_get_error_invalid_point ****") try: result = publish_agent.vip.rpc.call( PLATFORM_ACTUATOR, # Target agent @@ -1206,54 +1063,10 @@ def test_get_error_invalid_point(publish_agent): @pytest.mark.actuator -def test_set_value_float(publish_agent, cancel_schedules, revert_devices): - """ - Test setting a float value of a point through rpc - Expected result = value of the actuation point - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param cancel_schedules: fixture used to cancel the schedule at the end - of test so that other tests can use the same device and time slot - :param revert_devices: list of devices to revert during test +def test_revert_point_should_succeed(publish_agent, cancel_schedules): """ - print("\n**** test_set_float_value ****") - taskid = 'task_set_float_value' - agentid = TEST_AGENT - device = 'fakedriver0' - cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) - revert_devices.append({'agentid': agentid, 'device': device}) - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=2)) - msg = [ - [device, start, end] - ] - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == SUCCESS - - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver0/SampleWritableFloat1', # Point to set - 2.5 # New value - ).get(timeout=10) - assert result == 2.5 - + Test reverting a float value of a point through rpc using only the topic parameter -@pytest.mark.actuator -def test_revert_point(publish_agent, cancel_schedules): - """ - Test setting a float value of a point through rpc - Expected result = value of the actuation point :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing :param cancel_schedules: fixture used to cancel the schedule at the end @@ -1314,10 +1127,9 @@ def test_revert_point(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_revert_point_with_point(publish_agent, cancel_schedules): +def test_revert_point_with_point_should_succeed(publish_agent, cancel_schedules): """ - Test setting a float value of a point through rpc - Expected result = value of the actuation point + Test reverting a float value of a point through rpc using both topic and point parameters :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing @@ -1379,17 +1191,17 @@ def test_revert_point_with_point(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_revert_device(publish_agent, cancel_schedules): +def test_revert_device_should_succeed(publish_agent, cancel_schedules): """ - Test setting a float value of a point through rpc - Expected result = value of the actuation point + Tests whether a point is set to its initial value upon calling revert_device. + Consequently, this tests requires a lot of setup, namely setting a point to a new value, + verifying the change, then calling revert_device and again verifying that the point is set to its original value. :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing :param cancel_schedules: fixture used to cancel the schedule at the end of test so that other tests can use the same device and time slot """ - print ("\n**** test_set_float_value ****") taskid = 'test_revert_point' agentid = TEST_AGENT cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) @@ -1444,26 +1256,27 @@ def test_revert_device(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_set_error_array(publish_agent, cancel_schedules): +def test_set_point_should_succeed(publish_agent, cancel_schedules, revert_devices): """ - Test setting a array of single float value of a point. Should return - type error - + Test setting a float value of a point through rpc + Expected result = value of the actuation point :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing :param cancel_schedules: fixture used to cancel the schedule at the end of test so that other tests can use the same device and time slot + :param revert_devices: list of devices to revert during test """ - print("\n**** test_set_error_array ****") - # set agentid and task id for cancel_schedules fixture + print("\n**** test_set_float_value ****") + taskid = 'task_set_float_value' agentid = TEST_AGENT - taskid = 'task_set_float_array_value' + device = 'fakedriver0' cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) + revert_devices.append({'agentid': agentid, 'device': device}) start = str(datetime.now()) end = str(datetime.now() + timedelta(seconds=2)) msg = [ - ['fakedriver0', start, end] + [device, start, end] ] result = publish_agent.vip.rpc.call( PLATFORM_ACTUATOR, @@ -1475,22 +1288,64 @@ def test_set_error_array(publish_agent, cancel_schedules): # expected result {'info': u'', 'data': {}, 'result': SUCCESS} print(result) assert result['result'] == SUCCESS - try: - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver0/SampleWritableFloat1', # Point to set - [2.5] # New value - ).get(timeout=10) - pytest.fail('Expecting RemoteError for trying to set array on point ' + + result = publish_agent.vip.rpc.call( + PLATFORM_ACTUATOR, # Target agent + 'set_point', # Method + agentid, # Requestor + 'fakedriver0/SampleWritableFloat1', # Point to set + 2.5 # New value + ).get(timeout=10) + assert result == 2.5 + + +@pytest.mark.actuator +def test_set_point_raises_type_error_on_setting_array(publish_agent, cancel_schedules): + """ + Test setting a array of single float value of a point. Should return + type error + + :param publish_agent: fixture invoked to setup all agents necessary and + returns an instance of Agent object used for publishing + :param cancel_schedules: fixture used to cancel the schedule at the end + of test so that other tests can use the same device and time slot + """ + # set agentid and task id for cancel_schedules fixture + agentid = TEST_AGENT + taskid = 'task_set_float_array_value' + cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) + + start = str(datetime.now()) + end = str(datetime.now() + timedelta(seconds=2)) + msg = [ + ['fakedriver0', start, end] + ] + result = publish_agent.vip.rpc.call( + PLATFORM_ACTUATOR, + REQUEST_NEW_SCHEDULE, + agentid, + taskid, + PRIORITY_LOW, + msg).get(timeout=10) + # expected result {'info': u'', 'data': {}, 'result': SUCCESS} + print(result) + assert result['result'] == SUCCESS + try: + result = publish_agent.vip.rpc.call( + PLATFORM_ACTUATOR, # Target agent + 'set_point', # Method + agentid, # Requestor + 'fakedriver0/SampleWritableFloat1', # Point to set + [2.5] # New value + ).get(timeout=10) + pytest.fail('Expecting RemoteError for trying to set array on point ' 'that expects float. Code returned {}'.format(result)) except RemoteError as e: assert "TypeError" in e.message @pytest.mark.actuator -def test_set_lock_error(publish_agent): +def test_set_point_raises_lock_error(publish_agent): """ Test setting a float value of a point through rpc without an allocation Expected result @@ -1517,11 +1372,9 @@ def test_set_lock_error(publish_agent): @pytest.mark.actuator -def test_set_value_error(publish_agent, cancel_schedules): +def test_set_point_raises_value_error(publish_agent, cancel_schedules): """ Test setting a wrong type value of a point through rpc - - :param publish_agent: fixture invoked to setup all agents necessary and returns an instance of Agent object used for publishing :param cancel_schedules: fixture used to cancel the schedule at the end @@ -1562,16 +1415,7 @@ def test_set_value_error(publish_agent, cancel_schedules): @pytest.mark.actuator -def test_set_error_read_only_point(publish_agent, cancel_schedules): - """ - Test setting a value of a read only point through pubsub - - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param cancel_schedules: fixture used to cancel the schedule at the end - of test so that other tests can use the same device and time slot - """ - print ("\n**** test_set_read_only_point ****") +def test_set_point_raises_remote_error_on_read_only_point(publish_agent, cancel_schedules): agentid = TEST_AGENT taskid = 'task_set_readonly_point' cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) @@ -1593,46 +1437,147 @@ def test_set_error_read_only_point(publish_agent, cancel_schedules): print(result) assert result['result'] == SUCCESS - try: - result = publish_agent.vip.rpc.call( + with pytest.raises(RemoteError): + publish_agent.vip.rpc.call( 'platform.actuator', # Target agent 'set_point', # Method agentid, # Requestor 'fakedriver0/OutsideAirTemperature1', # Point to set 1.2 # New value ).get(timeout=10) - pytest.fail( - 'Expecting RemoteError but code returned: {}'.format(result)) - except RemoteError as e: - assert "RuntimeError" in e.message + pytest.fail("Expecting remote error.") @pytest.mark.actuator -def test_get_multiple_points(publish_agent, cancel_schedules): - results, errors = publish_agent.vip.rpc.call( - 'platform.actuator', - 'get_multiple_points', - ['fakedriver0/SampleWritableFloat1', - 'fakedriver1/SampleWritableFloat1']).get(timeout=10) +def test_set_point_should_succeed_on_allow_no_lock_write_default_setting(publish_agent, volttron_instance): + """ Tests the default setting, 'allow_no_lock_write=True', to allow writing without a + lock as long as nothing else has the device locked. - assert results == {'fakedriver0/SampleWritableFloat1': 10.0, 'fakedriver1/SampleWritableFloat1': 1.0} - assert errors == {} + :param publish_agent: fixture invoked to setup all agents necessary and + returns an instance of Agent object used for publishing + :param volttron_instance: Volttron instance on which test is run + """ + + alternate_actuator_vip_id = "my_actuator" + # Use actuator that allows write with no lock (i.e. allow_no_lock_write=True) + my_actuator_uuid = volttron_instance.install_agent( + agent_dir=get_services_core("ActuatorAgent"), + config_file=get_services_core("ActuatorAgent/tests/actuator-no-lock.config"), + start=True, vip_identity=alternate_actuator_vip_id) + agentid = "" + try: + agentid = TEST_AGENT + + result = publish_agent.vip.rpc.call( + alternate_actuator_vip_id, # Target agent + 'set_point', # Method + agentid, # Requestor + 'fakedriver0/SampleWritableFloat1', # Point to set + 6.5 # New value + ).get(timeout=10) + assert result == approx(6.5) + + finally: + publish_agent.vip.rpc.call( + alternate_actuator_vip_id, # Target agent + 'revert_device', # Method + agentid, # Requestor + 'fakedriver0' # Point to revert + ).get(timeout=10) + + volttron_instance.stop_agent(my_actuator_uuid) + volttron_instance.remove_agent(my_actuator_uuid) @pytest.mark.actuator -def test_get_multiple_points_separate_pointname(publish_agent, cancel_schedules): - results, errors = publish_agent.vip.rpc.call( - 'platform.actuator', - 'get_multiple_points', - [['fakedriver0', 'SampleWritableFloat1'], - ['fakedriver1', 'SampleWritableFloat1']]).get(timeout=10) +def test_set_point_raises_remote_error_on_allow_no_lock_write_default_setting(publish_agent, volttron_instance): + """ Tests the default setting, 'allow_no_lock_write=True', to allow writing without a + lock as long as nothing else has the device locked. In this case, we schedule the devices, thereby + creating a lock. Upon setting a point when a lock is created, this test should raise a RemoteError. - assert results == {'fakedriver0/SampleWritableFloat1': 10.0, 'fakedriver1/SampleWritableFloat1': 1.0} - assert errors == {} + :param publish_agent: fixture invoked to setup all agents necessary and + returns an instance of Agent object used for publishing + :param volttron_instance: Volttron instance on which test is run + """ + + alternate_actuator_vip_id = "my_actuator" + # Use actuator that allows write with no lock. + my_actuator_uuid = volttron_instance.install_agent( + agent_dir=get_services_core("ActuatorAgent"), + config_file=get_services_core("ActuatorAgent/tests/actuator-no-lock.config"), + start=True, vip_identity=alternate_actuator_vip_id) + publish_agent2 = None + try: + agentid2 = "test-agent2" + taskid = "test-task" + publish_agent2 = volttron_instance.build_agent(identity=agentid2) + + start = str(datetime.now()) + end = str(datetime.now() + timedelta(seconds=60)) + msg = [ + ['fakedriver0', start, end] + ] + result = publish_agent2.vip.rpc.call( + alternate_actuator_vip_id, + REQUEST_NEW_SCHEDULE, + agentid2, + taskid, + PRIORITY_LOW, + msg).get(timeout=10) + # expected result {'info': u'', 'data': {}, 'result': SUCCESS} + print(result) + assert result['result'] == SUCCESS + + agentid = TEST_AGENT + with pytest.raises(RemoteError): + publish_agent.vip.rpc.call( + alternate_actuator_vip_id, # Target agent + 'set_point', # Method + agentid, # Requestor + 'fakedriver0/SampleWritableFloat1', # Point to set + 7.5 # New value + ).get(timeout=10) + pytest.fail("Expecting remote error.") + + finally: + publish_agent2.vip.rpc.call( + alternate_actuator_vip_id, # Target agent + 'revert_device', # Method + agentid, # Requestor + 'fakedriver0' # Point to revert + ).get(timeout=10) + + publish_agent2.core.stop() + volttron_instance.stop_agent(my_actuator_uuid) + volttron_instance.remove_agent(my_actuator_uuid) + + +@pytest.mark.actuator +def test_set_point_raises_remote_error_on_lock_failure(publish_agent, cancel_schedules): + """ + Test setting a float value of a point through rpc + Expected result = value of the actuation point + :param publish_agent: fixture invoked to setup all agents necessary and + returns an instance of Agent object used for publishing + :param cancel_schedules: fixture used to cancel the schedule at the end + of test so that other tests can use the same device and time slot + """ + print ("\n**** test_set_float_value ****") + agentid = TEST_AGENT + + with pytest.raises(RemoteError): + publish_agent.vip.rpc.call( + PLATFORM_ACTUATOR, # Target agent + 'set_point', # Method + agentid, # Requestor + 'fakedriver0/SampleWritableFloat1', # Point to set + 2.5 # New value + ).get(timeout=10) + pytest.fail("Expecting remote error.") @pytest.mark.actuator -def test_get_multiple_captures_errors(publish_agent, cancel_schedules): +def test_get_multiple_points_captures_errors_on_nonexistent_point(publish_agent, cancel_schedules): results, errors = publish_agent.vip.rpc.call( 'platform.actuator', 'get_multiple_points', @@ -1643,16 +1588,31 @@ def test_get_multiple_captures_errors(publish_agent, cancel_schedules): "DriverInterfaceError('Point not configured on device: nonexistentpoint',)" +@pytest.mark.parametrize("invalid_topics, topic_key", [ + ([42], '42'), + ([None], 'None'), +]) @pytest.mark.actuator -def test_get_multiple_captures_errors_invalid_point(publish_agent, cancel_schedules): - results, errors = publish_agent.vip.rpc.call('platform.actuator', 'get_multiple_points', [42]).get(timeout=10) - +def test_get_multiple_points_captures_errors_on_invalid_topic(publish_agent, cancel_schedules, invalid_topics, topic_key): + results, errors = publish_agent.vip.rpc.call('platform.actuator', 'get_multiple_points', invalid_topics).get(timeout=10) assert results == {} - assert errors['42'] == "ValueError('Invalid topic: 42',)" - - + assert errors[topic_key] == f"ValueError('Invalid topic: {topic_key}',)" + + +@pytest.mark.parametrize( + "topics_values_list", + [ + ( + [('fakedriver0/SampleWritableFloat1', 42), + ('fakedriver1/SampleWritableFloat1', 42)] + ), + ( + [(('fakedriver0', 'SampleWritableFloat1'), 42), + (('fakedriver1', 'SampleWritableFloat1'), 42)] + ) + ]) @pytest.mark.actuator -def test_set_multiple_points(publish_agent, cancel_schedules): +def test_set_multiple_points_should_succeed(publish_agent, cancel_schedules, topics_values_list): agentid = TEST_AGENT taskid0 = 'task_point_on_device_0' taskid1 = 'task_point_on_device_1' @@ -1690,19 +1650,28 @@ def test_set_multiple_points(publish_agent, cancel_schedules): 'platform.actuator', 'set_multiple_points', agentid, - [('fakedriver0/SampleWritableFloat1', 42), - ('fakedriver1/SampleWritableFloat1', 42)]).get(timeout=10) + topics_values_list).get(timeout=10) assert result == {} @pytest.mark.actuator -def test_set_multiple_points_separate_pointname(publish_agent, cancel_schedules): +def test_set_multiple_points_raises_remote_error_on_no_lock(publish_agent, cancel_schedules): agentid = TEST_AGENT - taskid0 = 'task_point_on_device_0' - taskid1 = 'task_point_on_device_1' - cancel_schedules.append({'agentid': agentid, 'taskid': taskid0}) - cancel_schedules.append({'agentid': agentid, 'taskid': taskid1}) + with pytest.raises(RemoteError): + publish_agent.vip.rpc.call( + 'platform.actuator', + 'set_multiple_points', + agentid, + [('fakedriver0/SampleWritableFloat1', 42)]).get(timeout=10) + pytest.fail("Expecting remote error.") + + +@pytest.mark.actuator +def test_set_multiple_points_captures_errors_on_read_only_point(publish_agent, cancel_schedules): + agentid = TEST_AGENT + taskid = 'task_point_on_device_0' + cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) start = str(datetime.now()) end = str(datetime.now() + timedelta(seconds=2)) @@ -1714,19 +1683,7 @@ def test_set_multiple_points_separate_pointname(publish_agent, cancel_schedules) PLATFORM_ACTUATOR, REQUEST_NEW_SCHEDULE, agentid, - taskid0, - PRIORITY_LOW, - msg).get(timeout=10) - assert result['result'] == SUCCESS - - msg = [ - ['fakedriver1', start, end] - ] - result = publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, - REQUEST_NEW_SCHEDULE, - agentid, - taskid1, + taskid, PRIORITY_LOW, msg).get(timeout=10) assert result['result'] == SUCCESS @@ -1735,33 +1692,23 @@ def test_set_multiple_points_separate_pointname(publish_agent, cancel_schedules) 'platform.actuator', 'set_multiple_points', agentid, - [(('fakedriver0', 'SampleWritableFloat1'), 42), - (('fakedriver1', 'SampleWritableFloat1'), 42)]).get(timeout=10) - - assert result == {} - - -@pytest.mark.actuator -def test_set_multiple_raises_lock_error(publish_agent, cancel_schedules): - agentid = TEST_AGENT + [('fakedriver0/OutsideAirTemperature1', 42)]).get(timeout=10) try: - result = publish_agent.vip.rpc.call( - 'platform.actuator', - 'set_multiple_points', - agentid, - [('fakedriver0/SampleWritableFloat1', 42)]).get(timeout=10) + r = result['fakedriver0/OutsideAirTemperature1'] + assert "RuntimeError" in r + except KeyError: + pytest.fail('read only point did not raise an exception') - pytest.fail('Expecting LockError. Code returned: {}'.format(result)) - except Exception as e: - # TODO - check exc_info - assert e.exc_info['exc_type'].endswith("LockError") - assert e.message == \ - "caller ({}) does not lock for device {}".format(TEST_AGENT, 'fakedriver0') + assert True +@pytest.mark.parametrize("invalid_topics, topic_key", [ + (42, '42'), + (None, 'None'), +]) @pytest.mark.actuator -def test_set_multiple_captures_errors(publish_agent, cancel_schedules): +def test_set_multiple_points_captures_errors_on_invalid_topic(publish_agent, cancel_schedules, invalid_topics, topic_key): agentid = TEST_AGENT taskid = 'task_point_on_device_0' cancel_schedules.append({'agentid': agentid, 'taskid': taskid}) @@ -1785,154 +1732,25 @@ def test_set_multiple_captures_errors(publish_agent, cancel_schedules): 'platform.actuator', 'set_multiple_points', agentid, - [('fakedriver0/OutsideAirTemperature1', 42)]).get(timeout=10) + [(invalid_topics, 42.42)]).get(timeout=10) - try: - r = result['fakedriver0/OutsideAirTemperature1'] - assert "RuntimeError" in r - except KeyError: - pytest.fail('read only point did not raise an exception') - - assert True - - -@pytest.mark.actuator -def test_scrape_all(publish_agent, cancel_schedules): - result = publish_agent.vip.rpc.call('platform.actuator', 'scrape_all', 'fakedriver0').get(timeout=10) - assert type(result) is dict - assert len(result) == 13 + assert result[topic_key] == f"ValueError('Invalid topic: {topic_key}',)" @pytest.mark.actuator -def test_set_value_no_lock(publish_agent, volttron_instance): - """ Tests the (now default) setting to allow writing without a - lock as long as nothing else has the device locked. - - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param volttron_instance: Volttron instance on which test is run - """ - - alternate_actuator_vip_id = "my_actuator" - # Use actuator that allows write with no lock. - my_actuator_uuid = volttron_instance.install_agent( - agent_dir=get_services_core("ActuatorAgent"), - config_file=get_services_core("ActuatorAgent/tests/actuator-no-lock.config"), - start=True, vip_identity=alternate_actuator_vip_id) - agentid = "" - try: - agentid = TEST_AGENT +def test_scrape_all_should_succeed(publish_agent, cancel_schedules, volttron_instance): + points_filename = f"{volttron_instance.volttron_root}/scripts/scalability-testing/fake_unit_testing.csv" + with open(points_filename) as f: + expected_count_points = sum(1 for _ in f) - 1 - result = publish_agent.vip.rpc.call( - alternate_actuator_vip_id, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver0/SampleWritableFloat1', # Point to set - 6.5 # New value - ).get(timeout=10) - assert result == approx(6.5) - - finally: - publish_agent.vip.rpc.call( - alternate_actuator_vip_id, # Target agent - 'revert_device', # Method - agentid, # Requestor - 'fakedriver0' # Point to revert - ).get(timeout=10) - - volttron_instance.stop_agent(my_actuator_uuid) - volttron_instance.remove_agent(my_actuator_uuid) - - -@pytest.mark.actuator -def test_set_value_no_lock_failure(publish_agent, volttron_instance): - """ Tests the (now default) setting to allow writing without a - lock as long as nothing else has the device locked. - - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param volttron_instance: Volttron instance on which test is run - """ - - alternate_actuator_vip_id = "my_actuator" - # Use actuator that allows write with no lock. - my_actuator_uuid = volttron_instance.install_agent( - agent_dir=get_services_core("ActuatorAgent"), - config_file=get_services_core("ActuatorAgent/tests/actuator-no-lock.config"), - start=True, vip_identity=alternate_actuator_vip_id) - publish_agent2 = None - try: - agentid2 = "test-agent2" - taskid = "test-task" - publish_agent2 = volttron_instance.build_agent(identity=agentid2) - - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=60)) - msg = [ - ['fakedriver0', start, end] - ] - result = publish_agent2.vip.rpc.call( - alternate_actuator_vip_id, - REQUEST_NEW_SCHEDULE, - agentid2, - taskid, - PRIORITY_LOW, - msg).get(timeout=10) - # expected result {'info': u'', 'data': {}, 'result': SUCCESS} - print(result) - assert result['result'] == SUCCESS - - agentid = TEST_AGENT - - with pytest.raises(RemoteError): - publish_agent.vip.rpc.call( - alternate_actuator_vip_id, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver0/SampleWritableFloat1', # Point to set - 7.5 # New value - ).get(timeout=10) - pytest.fail("Expecting remote error.") - - finally: - publish_agent2.vip.rpc.call( - alternate_actuator_vip_id, # Target agent - 'revert_device', # Method - agentid, # Requestor - 'fakedriver0' # Point to revert - ).get(timeout=10) - - publish_agent2.core.stop() - volttron_instance.stop_agent(my_actuator_uuid) - volttron_instance.remove_agent(my_actuator_uuid) - - -@pytest.mark.actuator -def test_set_value_float_failure(publish_agent, cancel_schedules): - """ - Test setting a float value of a point through rpc - Expected result = value of the actuation point - :param publish_agent: fixture invoked to setup all agents necessary and - returns an instance of Agent object used for publishing - :param cancel_schedules: fixture used to cancel the schedule at the end - of test so that other tests can use the same device and time slot - """ - print ("\n**** test_set_float_value ****") - agentid = TEST_AGENT + result = publish_agent.vip.rpc.call('platform.actuator', 'scrape_all', 'fakedriver0').get(timeout=10) - with pytest.raises(RemoteError): - publish_agent.vip.rpc.call( - PLATFORM_ACTUATOR, # Target agent - 'set_point', # Method - agentid, # Requestor - 'fakedriver0/SampleWritableFloat1', # Point to set - 2.5 # New value - ).get(timeout=10) - pytest.fail("Expecting remote error.") + assert type(result) is dict + assert len(result) == expected_count_points @pytest.mark.actuator -def test_actuator_default_config(volttron_instance, publish_agent): +def test_actuator_default_config_should_succeed(volttron_instance, publish_agent): """ Test the default configuration file included with the agent """ diff --git a/services/core/ActuatorAgent/tests/test_actuator_rpc_unit.py b/services/core/ActuatorAgent/tests/test_actuator_rpc_unit.py deleted file mode 100644 index 38489bf6f3..0000000000 --- a/services/core/ActuatorAgent/tests/test_actuator_rpc_unit.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: -# -# Copyright 2020, Battelle Memorial Institute. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. -# -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 -# }}} - -""" -Unit test cases for testing actuator agent using rpc calls. -""" -import logging -from datetime import datetime, timedelta - -import pytest - -from services.core.ActuatorAgent.actuator import agent -from services.core.ActuatorAgent.actuator.agent import ActuatorAgent, LockError -from services.core.ActuatorAgent.actuator.scheduler import RequestResult, DeviceState -from services.core.ActuatorAgent.tests.actuator_fixtures import MockedAsyncResult, \ - get_actuator_agent -from volttrontesting.utils.utils import AgentMock -from volttron.platform.vip.agent import Agent - -pytestmark = [pytest.mark.actuator_unit, pytest.mark.unit] - -PRIORITY_LOW = "LOW" -SUCCESS = "SUCCESS" -FAILURE = "FAILURE" -REQUESTER_ID = "foo" -TASK_ID = "task-id" -TIME_SLOT_REQUESTS = [ - ["fakedriver0", str(datetime.now()), str(datetime.now() + timedelta(seconds=1))] -] - -agent._log = logging.getLogger("test_logger") -ActuatorAgent.__bases__ = (AgentMock.imitate(Agent, Agent()),) - - -@pytest.mark.parametrize("topic, point", [("path/topic", None), ("another/path/to/topic", 42)]) -def test_get_point_should_succeed(topic, point): - with get_actuator_agent(vip_rpc_call_res=MockedAsyncResult(10.0)) as actuator_agent: - result = actuator_agent.get_point(topic, point=point) - - actuator_agent.vip.rpc.call.assert_called_once() - assert result is not None - - -@pytest.mark.parametrize( - "point, device_state", - [ - ( - 42, - {"foo/bar": DeviceState("requester-id-1", "task-id-1", "anytime")}, - ), - ( - None, - {"foo": DeviceState("requester-id-1", "task-id-1", "anytime")}), - ], -) -def test_set_point_should_succeed(point, device_state): - requester_id = "requester-id-1" - topic = "foo/bar" - value = "some value" - - with get_actuator_agent(vip_message_peer=requester_id, device_state=device_state) as \ - actuator_agent: - result = actuator_agent.set_point(requester_id, topic, value, point=point) - - assert result is not None - - -@pytest.mark.parametrize("rpc_peer", [None, 42, []]) -def test_set_point_should_raise_type_error(rpc_peer): - with pytest.raises(TypeError, match="Agent id must be a nonempty string"): - requester_id = "requester-id-1" - topic = "foo/bar" - value = "some value" - point = None - - with get_actuator_agent(vip_message_peer=rpc_peer) as actuator_agent: - actuator_agent.set_point(requester_id, topic, value, point=point) - - -def test_set_point_should_raise_lock_error_on_non_matching_device(): - with pytest.raises(LockError): - requester_id = "requester-id-1" - topic = "foo/bar" - value = "some value" - - with get_actuator_agent(vip_message_peer="some rpc_peer") as actuator_agent: - actuator_agent.set_point(requester_id, topic, value) - - -def test_scrape_all_should_succeed(): - with get_actuator_agent(vip_rpc_call_res=MockedAsyncResult({})) as actuator_agent: - topic = "whan/that/aprille" - - result = actuator_agent.scrape_all(topic) - - assert isinstance(result, dict) - - - -@pytest.mark.parametrize( - "topics", - [ - ["foo/bar"], - ["foo/bar", "sna/foo"], - [["dev1", "point1"]], - [["dev1", "point1"], ["dev2", "point2"]], - ], -) -def test_get_multiple_points_should_succeed(topics): - mocked_rpc_call_res = MockedAsyncResult(({"result": "value"}, {})) - with get_actuator_agent(vip_rpc_call_res=mocked_rpc_call_res) as actuator_agent: - results, errors = actuator_agent.get_multiple_points(topics) - - assert isinstance(results, dict) - assert isinstance(errors, dict) - assert len(errors) == 0 - - -@pytest.mark.parametrize("invalid_topics", [[(123,)], [(None)], [[123]], [[None]]]) -def test_get_multiple_points_should_return_errors(invalid_topics): - with get_actuator_agent() as actuator_agent: - - results, errors = actuator_agent.get_multiple_points(invalid_topics) - - assert isinstance(results, dict) - assert isinstance(errors, dict) - assert len(errors) == 1 - - -@pytest.mark.parametrize( - "topic_values, device_state", - [ - ([], {}), - ( - [("foo/bar", "roma_value")], - {"foo": DeviceState("requester-id-1", "task-id-1", "anytime")}, - ), - ( - [("foo/bar", "roma_value"), ("sna/fu", "amor_value")], - { - "foo": DeviceState("requester-id-1", "task-id-1", "anytime"), - "sna": DeviceState("requester-id-1", "task-id-1", "anytime"), - }, - ), - ], -) -def test_set_multiple_points_should_succeed(topic_values, device_state): - requester_id = "requester-id-1" - mocked_rpc_call_res = MockedAsyncResult(({})) - with get_actuator_agent(vip_message_peer=requester_id, device_state=device_state, - vip_rpc_call_res=mocked_rpc_call_res) as actuator_agent: - result = actuator_agent.set_multiple_points("request-id-1", topic_values) - - assert result == {} - - -@pytest.mark.parametrize("invalid_topic_values", [[(None,)], [(1234,)]]) -def test_set_multiple_points_should_raise_value_error(invalid_topic_values): - with pytest.raises(ValueError): - requester_id = "requester-id-1" - - with get_actuator_agent(vip_message_peer=requester_id) as actuator_agent: - actuator_agent.set_multiple_points("request-id-1", invalid_topic_values) - - -def test_set_multiple_points_should_raise_lock_error_on_empty_devices(): - with pytest.raises(LockError): - requester_id = "requester-id-1" - topic_values = [("foo/bar", "roma_value")] - - with get_actuator_agent(vip_message_peer=requester_id) as actuator_agent: - actuator_agent.set_multiple_points("request-id-1", topic_values) - - -def test_set_multiple_points_should_raise_lock_error_on_non_matching_requester(): - with pytest.raises(LockError): - requester_id = "wrong-requester" - topic_values = [("foo/bar", "roma_value")] - device_state = { - "foo": DeviceState("requester-id-1", "task-id-1", "anytime") - } - - with get_actuator_agent(vip_message_peer=requester_id, device_state=device_state) \ - as actuator_agent: - actuator_agent.set_multiple_points("request-id-1", topic_values) - - -@pytest.mark.parametrize("point", [None, "foobarpoint"]) -def test_revert_point_should_raise_lock_error_on_empty_devices(point): - with pytest.raises(LockError): - requester_id = "request-id-1" - topic = "foo/bar" - - with get_actuator_agent(vip_message_peer="requester-id-1") as actuator_agent: - actuator_agent.revert_point(requester_id, topic, point=point) - - -@pytest.mark.parametrize("point", [None, "foobarpoint"]) -def test_revert_point_should_raise_lock_error_on_non_matching_requester(point): - with pytest.raises(LockError): - device_state = { - "foo": DeviceState("requester-id-1", "task-id-1", "anytime") - } - requester_id = "request-id-1" - topic = "foo/bar" - - with get_actuator_agent(vip_message_peer="wrong-requester", device_state=device_state) \ - as actuator_agent: - actuator_agent.revert_point(requester_id, topic, point=point) - - -def test_revert_device_should_raise_lock_error_on_empty_devices(): - with pytest.raises(LockError): - requester_id = "request-id-1" - topic = "foo/bar" - - with get_actuator_agent(vip_message_peer="requester-id-1") as actuator_agent: - actuator_agent.revert_device(requester_id, topic) - - -def test_revert_device_should_raise_lock_error_on_non_matching_requester(): - with pytest.raises(LockError): - device_state = { - "foo/bar": DeviceState("requester-id-1", "task-id-1", "anytime") - } - requester_id = "request-id-1" - topic = "foo/bar" - - with get_actuator_agent(vip_message_peer="wrong-requester", device_state=device_state) \ - as actuator_agent: - actuator_agent.revert_device(requester_id, topic) - - -def test_request_new_schedule_should_succeed(): - with get_actuator_agent() as actuator_agent: - result = actuator_agent.request_new_schedule(REQUESTER_ID, TASK_ID, - PRIORITY_LOW, TIME_SLOT_REQUESTS) - - assert result["result"] == SUCCESS - - -def test_request_new_schedule_should_succeed_when_stop_start_times_overlap(): - start = str(datetime.now()) - end = str(datetime.now() + timedelta(seconds=1)) - end2 = str(datetime.now() + timedelta(seconds=2)) - time_slot_requests = [["fakedriver0", start, end], ["fakedriver0", end, end2]] - - with get_actuator_agent() as actuator_agent: - result = actuator_agent.request_new_schedule(REQUESTER_ID, TASK_ID, - PRIORITY_LOW, time_slot_requests) - - assert result["result"] == SUCCESS - - -@pytest.mark.parametrize( - "task_id, expected_info", - [ - (1234, "MALFORMED_REQUEST: TypeError: taskid must be a nonempty string"), - ("", "MALFORMED_REQUEST: TypeError: taskid must be a nonempty string"), - (None, "MISSING_TASK_ID"), - ("task-id-duplicate", "TASK_ID_ALREADY_EXISTS"), - ], -) -def test_request_new_schedule_should_fail_on_invalid_taskid(task_id, expected_info): - false_request_result = RequestResult(False, {}, expected_info) - - with get_actuator_agent(slot_requests_res=false_request_result) as actuator_agent: - result = actuator_agent.request_new_schedule(REQUESTER_ID, task_id, - PRIORITY_LOW, TIME_SLOT_REQUESTS) - - assert result["result"] == FAILURE - assert result["info"] == expected_info - - -@pytest.mark.parametrize( - "invalid_priority, expected_info", - [("LOW2", "INVALID_PRIORITY"), (None, "MISSING_PRIORITY")], -) -def test_request_new_schedule_should_fail_on_invalid_priority(invalid_priority, expected_info): - false_request_result = RequestResult(False, {}, expected_info) - - with get_actuator_agent(slot_requests_res=false_request_result) as actuator_agent: - result = actuator_agent.request_new_schedule(REQUESTER_ID, TASK_ID, - invalid_priority, TIME_SLOT_REQUESTS) - - assert result["result"] == FAILURE - assert result["info"] == expected_info - - -@pytest.mark.parametrize( - "time_slot_request, expected_info", - [ - ( - [], - "MALFORMED_REQUEST_EMPTY"), - ( - [["fakedriver0", str(datetime.now()), ""]], - "MALFORMED_REQUEST: ParserError: String does not contain a date: ", - ), - ( - [["fakedriver0", str(datetime.now())]], - "MALFORMED_REQUEST: ValueError: " - "not enough values to unpack (expected 3, got 2)", - ), - ], -) -def test_request_new_schedule_should_fail_invalid_time_slot_requests(time_slot_request, - expected_info): - false_request_result = RequestResult(False, {}, expected_info) - - with get_actuator_agent(slot_requests_res=false_request_result) as actuator_agent: - result = actuator_agent.request_new_schedule( - REQUESTER_ID, TASK_ID, PRIORITY_LOW, time_slot_request - ) - - assert result["result"] == FAILURE - assert result["info"] == expected_info - - -def test_request_cancel_schedule_should_succeed_happy_path(): - true_request_result = RequestResult( - True, {}, "" - ) - - with get_actuator_agent(cancel_schedule_result=true_request_result) as actuator_agent: - result = actuator_agent.request_cancel_schedule(REQUESTER_ID, TASK_ID) - - assert result["result"] == SUCCESS - - -def test_request_cancel_schedule_should_fail_on_invalid_task_id(): - false_request_result = RequestResult( - False, {}, "TASK_ID_DOES_NOT_EXIST" - ) - invalid_task_id = "invalid-task-id" - - with get_actuator_agent(cancel_schedule_result=false_request_result) as actuator_agent: - result = actuator_agent.request_cancel_schedule(REQUESTER_ID, invalid_task_id) - - assert result["result"] == FAILURE - assert result["info"] == "TASK_ID_DOES_NOT_EXIST" diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/tests/test_chargepoint_driver.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/tests/test_chargepoint_driver.py index 61c7e70cc8..7851d675e4 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/tests/test_chargepoint_driver.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/tests/test_chargepoint_driver.py @@ -42,6 +42,8 @@ from volttron.platform import get_services_core +pytestmark = [pytest.mark.contrib] + DRIVER1_CONFIG_STRING = """{ "driver_config": { "stationID" : "1:34003", diff --git a/services/core/PlatformDriverAgent/tests/test_device_groups.py b/services/core/PlatformDriverAgent/tests/test_device_groups.py index b7e14b313d..2c306a48be 100644 --- a/services/core/PlatformDriverAgent/tests/test_device_groups.py +++ b/services/core/PlatformDriverAgent/tests/test_device_groups.py @@ -75,7 +75,7 @@ def add_result(self, peer, sender, bus, topic, headers, message): @pytest.fixture(scope="module") -def subscriber_agent(request, volttron_instance): +def subscriber_agent(volttron_instance): agent = volttron_instance.build_agent(identity='subscriber_agent', agent_class=_subscriber_agent) @@ -118,9 +118,10 @@ def subscriber_agent(request, volttron_instance): @pytest.fixture(scope="module") -def config_store_connection(request, volttron_instance): +def config_store_connection(volttron_instance): capabilities = [{'edit_config_store': {'identity': PLATFORM_DRIVER}}] connection = volttron_instance.build_connection(peer=CONFIGURATION_STORE, capabilities=capabilities) + gevent.sleep(1) # Reset platform driver config store connection.call("manage_delete_store", PLATFORM_DRIVER) @@ -141,14 +142,14 @@ def config_store_connection(request, volttron_instance): @pytest.fixture(scope="function") -def config_store(request, config_store_connection): +def config_store(config_store_connection): # Always have fake.csv ready to go. print("Adding fake.csv into store") config_store_connection.call("manage_store", PLATFORM_DRIVER, "fake.csv", registry_config_string, config_type="csv") yield config_store_connection - # Reset platform driver config store - print("Wiping out store.") + + print("Resetting platform driver config store") config_store_connection.call("manage_delete_store", PLATFORM_DRIVER) gevent.sleep(0.1) diff --git a/services/core/PlatformDriverAgent/tests/test_eagle.py b/services/core/PlatformDriverAgent/tests/test_eagle.py index 9dccc6d436..8856236739 100644 --- a/services/core/PlatformDriverAgent/tests/test_eagle.py +++ b/services/core/PlatformDriverAgent/tests/test_eagle.py @@ -45,6 +45,8 @@ from volttrontesting.utils.utils import get_rand_http_address from volttron.platform.agent.known_identities import CONFIGURATION_STORE, PLATFORM_DRIVER +pytestmark = [pytest.mark.skip(reason='Community-contributed driver'), pytest.mark.contrib] + server_addr = get_rand_http_address() no_scheme = server_addr[7:] ip, port = no_scheme.split(':') diff --git a/services/core/PlatformDriverAgent/tests/test_global_override.py b/services/core/PlatformDriverAgent/tests/test_global_override.py index 3e88b0bf44..486ddeada9 100644 --- a/services/core/PlatformDriverAgent/tests/test_global_override.py +++ b/services/core/PlatformDriverAgent/tests/test_global_override.py @@ -85,6 +85,7 @@ def config_store_connection(request, volttron_instance): capabilities = [{'edit_config_store': {'identity': PLATFORM_DRIVER}}] connection = volttron_instance.build_connection(peer=CONFIGURATION_STORE, capabilities=capabilities) + gevent.sleep(1) # Reset platform driver config store connection.call("manage_delete_store", PLATFORM_DRIVER) diff --git a/services/core/PlatformDriverAgent/tests/test_global_settings.py b/services/core/PlatformDriverAgent/tests/test_global_settings.py index aa116c8998..b6b3d0ad0a 100644 --- a/services/core/PlatformDriverAgent/tests/test_global_settings.py +++ b/services/core/PlatformDriverAgent/tests/test_global_settings.py @@ -152,6 +152,7 @@ def cleanup(): def config_store_connection(request, volttron_instance): capabilities = [{'edit_config_store': {'identity': PLATFORM_DRIVER}}] connection = volttron_instance.build_connection(peer=CONFIGURATION_STORE, capabilities=capabilities) + gevent.sleep(1) # Reset platform driver config store connection.call("manage_delete_store", PLATFORM_DRIVER) diff --git a/services/core/PlatformDriverAgent/tests/test_driver_bacnet_cov.py b/services/core/PlatformDriverAgent/tests/test_platform_driver_forward_cov.py similarity index 80% rename from services/core/PlatformDriverAgent/tests/test_driver_bacnet_cov.py rename to services/core/PlatformDriverAgent/tests/test_platform_driver_forward_cov.py index 18cecec127..bf10c871b4 100644 --- a/services/core/PlatformDriverAgent/tests/test_driver_bacnet_cov.py +++ b/services/core/PlatformDriverAgent/tests/test_platform_driver_forward_cov.py @@ -35,15 +35,15 @@ # BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY # under Contract DE-AC05-76RL01830 # }}} - import logging +import os import pytest import gevent import gevent.subprocess as subprocess -from gevent.subprocess import Popen +from gevent.subprocess import check_call from mock import MagicMock from volttron.platform.agent import utils -from volttron.platform import get_services_core +from volttron.platform import get_services_core, get_volttron_root from volttron.platform.agent.known_identities import PLATFORM_DRIVER utils.setup_logging() @@ -51,7 +51,7 @@ @pytest.fixture(scope="module") -def test_agent(request, volttron_instance): +def test_agent(volttron_instance): """Dynamic agent for sending rpc calls and listening to the bus""" agent = volttron_instance.build_agent() agent.cov_callback = MagicMock(name="cov_callback") @@ -62,36 +62,33 @@ def test_agent(request, volttron_instance): prefix="devices/fakedriver/all", callback=agent.cov_callback).get() - def stop_agent(): - print("In teardown method of query_agent") - agent.core.stop() + yield agent - request.addfinalizer(stop_agent) - return agent + _log.info("In teardown method of query_agent") + agent.core.stop() @pytest.mark.driver -def test_cov_update_published(volttron_instance, test_agent): +def test_forward_bacnet_cov_value(volttron_instance, test_agent): """Tests the functionality of BACnet change of value forwarding in the Platform Driver and driver.py""" # Reset platform driver config store cmd = ['volttron-ctl', 'config', 'delete', PLATFORM_DRIVER, '--all'] - process = Popen(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result = process.wait() - assert result == 0 + retcode = check_call(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert retcode == 0 # Add fake device configuration + fake_csv_infile = os.path.join(get_volttron_root(), 'examples/configurations/drivers/fake.csv') cmd = ['volttron-ctl', 'config', 'store', PLATFORM_DRIVER, - 'fake.csv', 'examples/configurations/drivers/fake.csv', '--csv'] - process = Popen(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result = process.wait() - assert result == 0 + 'fake.csv', fake_csv_infile, '--csv'] + retcode = check_call(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert retcode == 0 + fakedriver_infile = os.path.join(get_volttron_root(), 'examples/configurations/drivers/fake.config') cmd = ['volttron-ctl', 'config', 'store', PLATFORM_DRIVER, - "devices/fakedriver", 'examples/configurations/drivers/fake.config', '--json'] - process = Popen(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result = process.wait() - assert result == 0 + "devices/fakedriver", fakedriver_infile, '--json'] + retcode = check_call(cmd, env=volttron_instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert retcode == 0 # install platform driver, start the platform driver, which starts the device platform_uuid = volttron_instance.install_agent( diff --git a/services/core/VolttronCentral/volttroncentral/agent.py b/services/core/VolttronCentral/volttroncentral/agent.py index a8caa5bdc3..19e6eb8da6 100644 --- a/services/core/VolttronCentral/volttroncentral/agent.py +++ b/services/core/VolttronCentral/volttroncentral/agent.py @@ -412,7 +412,7 @@ def jsonrpc(self, env, data): resp = requests.post(auth_url, json=args, verify=False) if resp.ok and resp.text: - claims = self.vip.web.get_user_claims(resp.text) + claims = self.vip.web.get_user_claims(jsonapi.loads(resp.text)["access_token"]) # Because the web-user.json has the groups under a key and the # groups is just passed into the session we need to make sure # we pass in the proper thing to the _add_sesion function. diff --git a/services/core/MongodbHistorian/README.md b/services/unsupported/MongodbHistorian/README.md similarity index 100% rename from services/core/MongodbHistorian/README.md rename to services/unsupported/MongodbHistorian/README.md diff --git a/services/core/MongodbHistorian/config b/services/unsupported/MongodbHistorian/config similarity index 100% rename from services/core/MongodbHistorian/config rename to services/unsupported/MongodbHistorian/config diff --git a/services/core/MongodbHistorian/conftest.py b/services/unsupported/MongodbHistorian/conftest.py similarity index 100% rename from services/core/MongodbHistorian/conftest.py rename to services/unsupported/MongodbHistorian/conftest.py diff --git a/services/core/OpenADRVenAgent/__init__.py b/services/unsupported/MongodbHistorian/mongodb/__init__.py similarity index 100% rename from services/core/OpenADRVenAgent/__init__.py rename to services/unsupported/MongodbHistorian/mongodb/__init__.py diff --git a/services/core/MongodbHistorian/mongodb/historian.py b/services/unsupported/MongodbHistorian/mongodb/historian.py similarity index 100% rename from services/core/MongodbHistorian/mongodb/historian.py rename to services/unsupported/MongodbHistorian/mongodb/historian.py diff --git a/services/core/MongodbHistorian/requirements.txt b/services/unsupported/MongodbHistorian/requirements.txt similarity index 100% rename from services/core/MongodbHistorian/requirements.txt rename to services/unsupported/MongodbHistorian/requirements.txt diff --git a/services/core/MongodbHistorian/scripts/count_data_by_topic_pattern.py b/services/unsupported/MongodbHistorian/scripts/count_data_by_topic_pattern.py similarity index 100% rename from services/core/MongodbHistorian/scripts/count_data_by_topic_pattern.py rename to services/unsupported/MongodbHistorian/scripts/count_data_by_topic_pattern.py diff --git a/services/core/MongodbHistorian/scripts/rollup_data_by_time.py b/services/unsupported/MongodbHistorian/scripts/rollup_data_by_time.py similarity index 100% rename from services/core/MongodbHistorian/scripts/rollup_data_by_time.py rename to services/unsupported/MongodbHistorian/scripts/rollup_data_by_time.py diff --git a/services/core/MongodbHistorian/scripts/rollup_data_using_groupby.py b/services/unsupported/MongodbHistorian/scripts/rollup_data_using_groupby.py similarity index 100% rename from services/core/MongodbHistorian/scripts/rollup_data_using_groupby.py rename to services/unsupported/MongodbHistorian/scripts/rollup_data_using_groupby.py diff --git a/services/core/MongodbHistorian/setup.py b/services/unsupported/MongodbHistorian/setup.py similarity index 100% rename from services/core/MongodbHistorian/setup.py rename to services/unsupported/MongodbHistorian/setup.py diff --git a/services/core/MongodbHistorian/tests/fixtures.py b/services/unsupported/MongodbHistorian/tests/fixtures.py similarity index 100% rename from services/core/MongodbHistorian/tests/fixtures.py rename to services/unsupported/MongodbHistorian/tests/fixtures.py diff --git a/services/core/MongodbHistorian/tests/mongod.conf b/services/unsupported/MongodbHistorian/tests/mongod.conf similarity index 100% rename from services/core/MongodbHistorian/tests/mongod.conf rename to services/unsupported/MongodbHistorian/tests/mongod.conf diff --git a/services/core/MongodbHistorian/tests/mongosetup.sh b/services/unsupported/MongodbHistorian/tests/mongosetup.sh similarity index 100% rename from services/core/MongodbHistorian/tests/mongosetup.sh rename to services/unsupported/MongodbHistorian/tests/mongosetup.sh diff --git a/services/core/MongodbHistorian/tests/purgemongo.sh b/services/unsupported/MongodbHistorian/tests/purgemongo.sh similarity index 100% rename from services/core/MongodbHistorian/tests/purgemongo.sh rename to services/unsupported/MongodbHistorian/tests/purgemongo.sh diff --git a/services/core/MongodbHistorian/tests/test_mongohistorian.py b/services/unsupported/MongodbHistorian/tests/test_mongohistorian.py similarity index 100% rename from services/core/MongodbHistorian/tests/test_mongohistorian.py rename to services/unsupported/MongodbHistorian/tests/test_mongohistorian.py diff --git a/services/core/MongodbHistorian/tests/test_prod_query_mongo.py b/services/unsupported/MongodbHistorian/tests/test_prod_query_mongo.py similarity index 100% rename from services/core/MongodbHistorian/tests/test_prod_query_mongo.py rename to services/unsupported/MongodbHistorian/tests/test_prod_query_mongo.py diff --git a/services/core/OpenADRVenAgent/IDENTITY b/services/unsupported/OpenADRVenAgent/IDENTITY similarity index 100% rename from services/core/OpenADRVenAgent/IDENTITY rename to services/unsupported/OpenADRVenAgent/IDENTITY diff --git a/services/core/OpenADRVenAgent/README.md b/services/unsupported/OpenADRVenAgent/README.md similarity index 100% rename from services/core/OpenADRVenAgent/README.md rename to services/unsupported/OpenADRVenAgent/README.md diff --git a/services/core/OpenADRVenAgent/openadrven/__init__.py b/services/unsupported/OpenADRVenAgent/__init__.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/__init__.py rename to services/unsupported/OpenADRVenAgent/__init__.py diff --git a/services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_BOTH0002_Cert.pem b/services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_BOTH0002_Cert.pem similarity index 100% rename from services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_BOTH0002_Cert.pem rename to services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_BOTH0002_Cert.pem diff --git a/services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_MCA0002_Cert.pem b/services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_MCA0002_Cert.pem similarity index 100% rename from services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_MCA0002_Cert.pem rename to services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_MCA0002_Cert.pem diff --git a/services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_RCA0002_Cert.pem b/services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_RCA0002_Cert.pem similarity index 100% rename from services/core/OpenADRVenAgent/certs/TEST_OpenADR_RSA_RCA0002_Cert.pem rename to services/unsupported/OpenADRVenAgent/certs/TEST_OpenADR_RSA_RCA0002_Cert.pem diff --git a/services/core/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_cert.pem b/services/unsupported/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_cert.pem similarity index 100% rename from services/core/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_cert.pem rename to services/unsupported/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_cert.pem diff --git a/services/core/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_privkey.pem b/services/unsupported/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_privkey.pem similarity index 100% rename from services/core/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_privkey.pem rename to services/unsupported/OpenADRVenAgent/certs/TEST_RSA_VEN_171024145702_privkey.pem diff --git a/services/core/OpenADRVenAgent/config b/services/unsupported/OpenADRVenAgent/config similarity index 100% rename from services/core/OpenADRVenAgent/config rename to services/unsupported/OpenADRVenAgent/config diff --git a/services/core/OpenADRVenAgent/conftest.py b/services/unsupported/OpenADRVenAgent/conftest.py similarity index 100% rename from services/core/OpenADRVenAgent/conftest.py rename to services/unsupported/OpenADRVenAgent/conftest.py diff --git a/services/core/OpenADRVenAgent/install-ven-agent.sh b/services/unsupported/OpenADRVenAgent/install-ven-agent.sh similarity index 100% rename from services/core/OpenADRVenAgent/install-ven-agent.sh rename to services/unsupported/OpenADRVenAgent/install-ven-agent.sh diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/__init__.py b/services/unsupported/OpenADRVenAgent/openadrven/__init__.py similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/__init__.py rename to services/unsupported/OpenADRVenAgent/openadrven/__init__.py diff --git a/services/core/OpenADRVenAgent/openadrven/agent.py b/services/unsupported/OpenADRVenAgent/openadrven/agent.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/agent.py rename to services/unsupported/OpenADRVenAgent/openadrven/agent.py diff --git a/services/core/OpenADRVenAgent/openadrven/models.py b/services/unsupported/OpenADRVenAgent/openadrven/models.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/models.py rename to services/unsupported/OpenADRVenAgent/openadrven/models.py diff --git a/services/core/OpenADRVenAgent/openadrven/oadr_20b.py b/services/unsupported/OpenADRVenAgent/openadrven/oadr_20b.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/oadr_20b.py rename to services/unsupported/OpenADRVenAgent/openadrven/oadr_20b.py diff --git a/services/core/OpenADRVenAgent/openadrven/oadr_builder.py b/services/unsupported/OpenADRVenAgent/openadrven/oadr_builder.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/oadr_builder.py rename to services/unsupported/OpenADRVenAgent/openadrven/oadr_builder.py diff --git a/services/core/OpenADRVenAgent/openadrven/oadr_common.py b/services/unsupported/OpenADRVenAgent/openadrven/oadr_common.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/oadr_common.py rename to services/unsupported/OpenADRVenAgent/openadrven/oadr_common.py diff --git a/services/core/OpenADRVenAgent/openadrven/oadr_extractor.py b/services/unsupported/OpenADRVenAgent/openadrven/oadr_extractor.py similarity index 100% rename from services/core/OpenADRVenAgent/openadrven/oadr_extractor.py rename to services/unsupported/OpenADRVenAgent/openadrven/oadr_extractor.py diff --git a/services/core/OpenADRVenAgent/requirements.txt b/services/unsupported/OpenADRVenAgent/requirements.txt similarity index 100% rename from services/core/OpenADRVenAgent/requirements.txt rename to services/unsupported/OpenADRVenAgent/requirements.txt diff --git a/services/ops/FailoverAgent/setup.py b/services/unsupported/OpenADRVenAgent/setup.py similarity index 100% rename from services/ops/FailoverAgent/setup.py rename to services/unsupported/OpenADRVenAgent/setup.py diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/IDENTITY b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/IDENTITY similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/IDENTITY rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/IDENTITY diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/__init__.py b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/__init__.py similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/__init__.py rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/__init__.py diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim.config b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim.config similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim.config rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim.config diff --git a/services/core/OpenADRVenAgent/test/__init__.py b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/__init__.py similarity index 100% rename from services/core/OpenADRVenAgent/test/__init__.py rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/__init__.py diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/agent.py b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/agent.py similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/agent.py rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/controlagentsim/agent.py diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/install-control-agent.sh b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/install-control-agent.sh similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/install-control-agent.sh rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/install-control-agent.sh diff --git a/services/core/OpenADRVenAgent/test/ControlAgentSim/setup.py b/services/unsupported/OpenADRVenAgent/test/ControlAgentSim/setup.py similarity index 100% rename from services/core/OpenADRVenAgent/test/ControlAgentSim/setup.py rename to services/unsupported/OpenADRVenAgent/test/ControlAgentSim/setup.py diff --git a/services/ops/FailoverAgent/failover/__init__.py b/services/unsupported/OpenADRVenAgent/test/__init__.py similarity index 100% rename from services/ops/FailoverAgent/failover/__init__.py rename to services/unsupported/OpenADRVenAgent/test/__init__.py diff --git a/services/core/OpenADRVenAgent/test/crypto_experiment.py b/services/unsupported/OpenADRVenAgent/test/crypto_experiment.py similarity index 100% rename from services/core/OpenADRVenAgent/test/crypto_experiment.py rename to services/unsupported/OpenADRVenAgent/test/crypto_experiment.py diff --git a/services/core/OpenADRVenAgent/test/curl_event.sh b/services/unsupported/OpenADRVenAgent/test/curl_event.sh similarity index 100% rename from services/core/OpenADRVenAgent/test/curl_event.sh rename to services/unsupported/OpenADRVenAgent/test/curl_event.sh diff --git a/services/core/OpenADRVenAgent/test/curl_report.sh b/services/unsupported/OpenADRVenAgent/test/curl_report.sh similarity index 100% rename from services/core/OpenADRVenAgent/test/curl_report.sh rename to services/unsupported/OpenADRVenAgent/test/curl_report.sh diff --git a/services/core/OpenADRVenAgent/test/curl_vtn_poll.sh b/services/unsupported/OpenADRVenAgent/test/curl_vtn_poll.sh similarity index 100% rename from services/core/OpenADRVenAgent/test/curl_vtn_poll.sh rename to services/unsupported/OpenADRVenAgent/test/curl_vtn_poll.sh diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/dkuhlman-generateds-cba0ef052d1e.zip b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/dkuhlman-generateds-cba0ef052d1e.zip similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/dkuhlman-generateds-cba0ef052d1e.zip rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/dkuhlman-generateds-cba0ef052d1e.zip diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.0.tar.gz b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.0.tar.gz similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.0.tar.gz rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.0.tar.gz diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3-old.tar.gz b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3-old.tar.gz similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3-old.tar.gz rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3-old.tar.gz diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3.tar.gz b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3.tar.gz similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3.tar.gz rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.3.tar.gz diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7.tar.gz b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7.tar.gz similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7.tar.gz rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7.tar.gz diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7_unpatched.tar.gz b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7_unpatched.tar.gz similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7_unpatched.tar.gz rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/generateDS-2.29.7_unpatched.tar.gz diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_0_original.py b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_0_original.py similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_0_original.py rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_0_original.py diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_3_original.py b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_3_original.py similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_3_original.py rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_3_original.py diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_4_original.py b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_4_original.py similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_4_original.py rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_20b_v2_29_4_original.py diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_ISO_ISO3AlphaCurrencyCode_20100407.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_ISO_ISO3AlphaCurrencyCode_20100407.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_ISO_ISO3AlphaCurrencyCode_20100407.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_ISO_ISO3AlphaCurrencyCode_20100407.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_atom.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_atom.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_atom.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_atom.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_ei_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_ei_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_ei_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_ei_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_emix_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_emix_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_emix_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_emix_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_gml_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_gml_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_gml_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_gml_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_greenbutton.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_greenbutton.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_greenbutton.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_greenbutton.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_power_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_power_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_power_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_power_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_pyld_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_pyld_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_pyld_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_pyld_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_siscale_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_siscale_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_siscale_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_siscale_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_strm_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_strm_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_strm_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_strm_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xcal_20b.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xcal_20b.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xcal_20b.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xcal_20b.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xml.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xml.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xml.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xml.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig-properties-schema.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig-properties-schema.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig-properties-schema.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig-properties-schema.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig.xsd diff --git a/services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig11.xsd b/services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig11.xsd similarity index 100% rename from services/core/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig11.xsd rename to services/unsupported/OpenADRVenAgent/test/oadr20b_schema/oadr_xmldsig11.xsd diff --git a/services/core/OpenADRVenAgent/test/test_ven.py b/services/unsupported/OpenADRVenAgent/test/test_ven.py similarity index 100% rename from services/core/OpenADRVenAgent/test/test_ven.py rename to services/unsupported/OpenADRVenAgent/test/test_ven.py diff --git a/services/core/OpenADRVenAgent/test/xml/crypto_experiment.xml b/services/unsupported/OpenADRVenAgent/test/xml/crypto_experiment.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/crypto_experiment.xml rename to services/unsupported/OpenADRVenAgent/test/xml/crypto_experiment.xml diff --git a/services/core/OpenADRVenAgent/test/xml/epri_vtn_distribute_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_distribute_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/epri_vtn_distribute_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_distribute_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/epri_vtn_response_not_registered.xml b/services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_response_not_registered.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/epri_vtn_response_not_registered.xml rename to services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_response_not_registered.xml diff --git a/services/core/OpenADRVenAgent/test/xml/epri_vtn_response_valid.xml b/services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_response_valid.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/epri_vtn_response_valid.xml rename to services/unsupported/OpenADRVenAgent/test/xml/epri_vtn_response_valid.xml diff --git a/services/core/OpenADRVenAgent/test/xml/kisensum_vtn_cancel_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_cancel_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/kisensum_vtn_cancel_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_cancel_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/kisensum_vtn_distribute_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_distribute_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/kisensum_vtn_distribute_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_distribute_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/kisensum_vtn_registered_report.xml b/services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_registered_report.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/kisensum_vtn_registered_report.xml rename to services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_registered_report.xml diff --git a/services/core/OpenADRVenAgent/test/xml/kisensum_vtn_response.xml b/services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_response.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/kisensum_vtn_response.xml rename to services/unsupported/OpenADRVenAgent/test/xml/kisensum_vtn_response.xml diff --git a/services/core/OpenADRVenAgent/test/xml/sample_ven_poll_signed.xml b/services/unsupported/OpenADRVenAgent/test/xml/sample_ven_poll_signed.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/sample_ven_poll_signed.xml rename to services/unsupported/OpenADRVenAgent/test/xml/sample_ven_poll_signed.xml diff --git a/services/core/OpenADRVenAgent/test/xml/test_vtn_cancel_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/test_vtn_cancel_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/test_vtn_cancel_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/test_vtn_cancel_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event_no_end.xml b/services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event_no_end.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event_no_end.xml rename to services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event_no_end.xml diff --git a/services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event_variable.xml b/services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event_variable.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/test_vtn_distribute_event_variable.xml rename to services/unsupported/OpenADRVenAgent/test/xml/test_vtn_distribute_event_variable.xml diff --git a/services/core/OpenADRVenAgent/test/xml/test_vtn_registered_report.xml b/services/unsupported/OpenADRVenAgent/test/xml/test_vtn_registered_report.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/test_vtn_registered_report.xml rename to services/unsupported/OpenADRVenAgent/test/xml/test_vtn_registered_report.xml diff --git a/services/core/OpenADRVenAgent/test/xml/ven_created_event.xml b/services/unsupported/OpenADRVenAgent/test/xml/ven_created_event.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/ven_created_event.xml rename to services/unsupported/OpenADRVenAgent/test/xml/ven_created_event.xml diff --git a/services/core/OpenADRVenAgent/test/xml/ven_created_report.xml b/services/unsupported/OpenADRVenAgent/test/xml/ven_created_report.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/ven_created_report.xml rename to services/unsupported/OpenADRVenAgent/test/xml/ven_created_report.xml diff --git a/services/core/OpenADRVenAgent/test/xml/ven_poll.xml b/services/unsupported/OpenADRVenAgent/test/xml/ven_poll.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/ven_poll.xml rename to services/unsupported/OpenADRVenAgent/test/xml/ven_poll.xml diff --git a/services/core/OpenADRVenAgent/test/xml/ven_register_report.xml b/services/unsupported/OpenADRVenAgent/test/xml/ven_register_report.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/ven_register_report.xml rename to services/unsupported/OpenADRVenAgent/test/xml/ven_register_report.xml diff --git a/services/core/OpenADRVenAgent/test/xml/ven_update_report.xml b/services/unsupported/OpenADRVenAgent/test/xml/ven_update_report.xml similarity index 100% rename from services/core/OpenADRVenAgent/test/xml/ven_update_report.xml rename to services/unsupported/OpenADRVenAgent/test/xml/ven_update_report.xml diff --git a/volttron/platform/certs.py b/volttron/platform/certs.py index b6d2725cbb..e4c73779cc 100644 --- a/volttron/platform/certs.py +++ b/volttron/platform/certs.py @@ -43,7 +43,7 @@ import six import time from shutil import copyfile -from socket import gethostname, getfqdn +from socket import gethostname, getfqdn, getaddrinfo, AI_CANONNAME import subprocess from cryptography import x509 @@ -958,12 +958,11 @@ def _create_signed_certificate(ca_cert, ca_key, name, valid_days=DEFAULT_DAYS, t for i in temp_list: if i.get_attributes_for_oid(NameOID.COMMON_NAME): if type == 'server': - # TODO: Also add SubjectAltName if fqdn: - hostname = fqdn + hostname = gethostname() else: - hostname = getfqdn() - fqdn = hostname + hostname = gethostname() + fqdn = getfqdn(hostname) new_attrs.append(RelativeDistinguishedName( [x509.NameAttribute( NameOID.COMMON_NAME, @@ -1025,10 +1024,17 @@ def _create_signed_certificate(ca_cert, ca_key, name, valid_days=DEFAULT_DAYS, t x509.ExtendedKeyUsage((ExtendedKeyUsageOID.SERVER_AUTH,)), critical=False ) - cert_builder = cert_builder.add_extension( - x509.SubjectAlternativeName((DNSName(fqdn),)), - critical=True + if hostname and fqdn != hostname: + cert_builder = cert_builder.add_extension( + x509.SubjectAlternativeName([DNSName(hostname), DNSName(fqdn)]), + critical=True + ) + else: + cert_builder = cert_builder.add_extension( + x509.SubjectAlternativeName([DNSName(fqdn)]), + critical=True ) + elif type == 'client': # specify that the certificate can be used as an SSL # client certificate to enable TLS Web Client Authentication diff --git a/volttron/platform/dbutils/postgresqlfuncts.py b/volttron/platform/dbutils/postgresqlfuncts.py index 39a69c5da6..af964a88b7 100644 --- a/volttron/platform/dbutils/postgresqlfuncts.py +++ b/volttron/platform/dbutils/postgresqlfuncts.py @@ -121,7 +121,8 @@ def setup_historian_tables(self): "SELECT create_hypertable({}, 'ts', if_not_exists => true)").format( Literal(self.data_table))) self.execute_stmt(SQL( - 'CREATE INDEX ON {} (topic_id, ts)').format( + 'CREATE INDEX IF NOT EXISTS {} ON {} (topic_id, ts)').format( + Identifier('idx_' + self.data_table), Identifier(self.data_table))) else: self.execute_stmt(SQL( diff --git a/volttron/platform/vip/pubsubservice.py b/volttron/platform/vip/pubsubservice.py index 732ebb5faa..917eeae85d 100644 --- a/volttron/platform/vip/pubsubservice.py +++ b/volttron/platform/vip/pubsubservice.py @@ -493,6 +493,7 @@ def _distribute_external(self, frames): frames = [publisher, '', proto, user_id, msg_id, 'error', errnum, errmsg, platform_id, subsystem] try: + frames = serialize_frames(frames) self._vip_sock.send_multipart(frames, flags=NOBLOCK, copy=False) except ZMQError as exc: # raise @@ -543,9 +544,10 @@ def _send(self, frames, publisher): self._logger.debug("EAGAIN error {}".format(subscriber)) # Only send EAGAIN errors proto, user_id, msg_id, subsystem = frames[2:6] - frames = [publisher, b'', proto, user_id, msg_id, - b'error', errnum, errmsg, subscriber, subsystem] + frames = [publisher, '', proto, user_id, msg_id, + 'error', errnum, errmsg, subscriber, subsystem] try: + frames = serialize_frames(frames) self._vip_sock.send_multipart(frames, flags=NOBLOCK, copy=False) except ZMQError as exc: # raise @@ -790,8 +792,8 @@ def _external_to_local_publish(self, frames): # peer is not authorized to publish to the topic, send error message to the peer if errmsg is not None: try: - frames = [publisher, b'', proto, user_id, msg_id, - subsystem, b'error', zmq.Frame(str(UNAUTHORIZED).encode("utf-8")), + frames = [publisher, '', proto, user_id, msg_id, + subsystem, 'error', zmq.Frame(str(UNAUTHORIZED).encode("utf-8")), zmq.Frame(str(errmsg).encode("utf-8"))] self._ext_router.send_external(publisher, frames) return @@ -808,7 +810,7 @@ def _external_to_local_publish(self, frames): # There are no subscribers, send error message back to source platform if not subscribers_count: try: - frames = [publisher, b'', proto, user_id, msg_id, + frames = [publisher, '', proto, user_id, msg_id, subsystem, zmq.Frame(b'error'), zmq.Frame(str(INVALID_REQUEST).encode("utf-8")), topic] self._ext_router.send_external(publisher, frames) diff --git a/volttron/platform/web/__init__.py b/volttron/platform/web/__init__.py index 099f606ef7..c9bffba28a 100644 --- a/volttron/platform/web/__init__.py +++ b/volttron/platform/web/__init__.py @@ -39,6 +39,7 @@ from http.cookies import SimpleCookie import logging +from datetime import datetime from volttron.platform import get_platform_config try: @@ -66,6 +67,8 @@ def get_bearer(env): auth_type, bearer = http_auth.split(' ') if auth_type.upper() != 'BEARER': raise NotAuthorized("Invalid HTTP_AUTHORIZATION header passed, must be Bearer") + else: + return bearer else: cookiestr = env.get('HTTP_COOKIE') if not cookiestr: @@ -110,7 +113,7 @@ def __get_key_and_algorithm__(env, ssl_public_key): def get_user_claim_from_bearer(bearer, web_secret_key=None, tls_public_key=None): if web_secret_key is None and tls_public_key is None: raise ValueError("web_secret_key or tls_public_key must be set") - if web_secret_key is None and tls_public_key is None: + if web_secret_key is not None and tls_public_key is not None: raise ValueError("web_secret_key or tls_public_key must be set not both") if web_secret_key is not None: @@ -121,7 +124,6 @@ def get_user_claim_from_bearer(bearer, web_secret_key=None, tls_public_key=None) pubkey = tls_public_key # if isinstance(tls_public_key, str): # pubkey = CertWrapper.load_cert(tls_public_key) - return jwt.decode(bearer, pubkey, algorithms=algorithm) - - + claims = jwt.decode(bearer, pubkey, algorithms=algorithm) + return claims diff --git a/volttron/platform/web/authenticate_endpoint.py b/volttron/platform/web/authenticate_endpoint.py index 0af6ff425f..ebfe561194 100644 --- a/volttron/platform/web/authenticate_endpoint.py +++ b/volttron/platform/web/authenticate_endpoint.py @@ -2,6 +2,8 @@ import os import re from urllib.parse import parse_qs +from datetime import datetime, timedelta +import json import jwt from jinja2 import Environment, FileSystemLoader, select_autoescape, TemplateNotFound @@ -32,9 +34,12 @@ class AuthenticateEndpoints(object): - def __init__(self, tls_private_key=None, web_secret_key=None): + def __init__(self, tls_private_key=None, tls_public_key=None, web_secret_key=None): + self.refresh_token_timeout = 240 # minutes before token expires. TODO: Should this be a setting somewhere? + self.access_token_timeout = 15 # minutes before token expires. TODO: Should this be a setting somewhere? self._tls_private_key = tls_private_key + self._tls_public_key = tls_public_key self._web_secret_key = web_secret_key if self._tls_private_key is None and self._web_secret_key is None: raise ValueError("Must have either ssl_private_key or web_secret_key specified!") @@ -72,21 +77,44 @@ def get_routes(self): :return: """ return [ - (re.compile('^/authenticate'), 'callable', self.get_auth_token) + (re.compile('^/authenticate'), 'callable', self.handle_authenticate) ] - def get_auth_token(self, env, data): + def handle_authenticate(self, env, data): """ - Creates an authentication token to be returned to the caller. The - response will be a text/plain encoded user + Callback for /authenticate endpoint. + Routes request based on HTTP method and returns a text/plain encoded token or error. + + :param env: + :param data: + :return: Response + """ + method = env.get('REQUEST_METHOD') + if method == 'POST': + response = self.get_auth_tokens(env, data) + elif method == 'PUT': + response = self.renew_auth_token(env, data) + elif method == 'DELETE': + response = self.revoke_auth_token(env, data) + else: + error = f"/authenticate endpoint accepts only POST, PUT, or DELETE methods. Received: {method}" + _log.warning(error) + return Response(error, status='405 Method Not Allowed', content_type='text/plain') + return response + + def get_auth_tokens(self, env, data): + """ + Creates an authentication refresh and acccss tokens to be returned to the caller. The + response will be a text/plain encoded user. Data should contain: + { + "username": "", + "password": "" + } :param env: :param data: :return: """ - if env.get('REQUEST_METHOD') != 'POST': - _log.warning("Authentication must use POST request.") - return Response('401 Unauthorized', status='401 Unauthorized', content_type='text/html') assert len(self._userdict) > 0, "No users in user dictionary, set the administrator password first!" @@ -118,12 +146,67 @@ def get_auth_token(self, env, data): if user is None: _log.error("No matching user for passed username: {}".format(username)) return Response('', status='401') - + access_token, refresh_token = self._get_tokens(user) + response = Response(json.dumps({"refresh_token": refresh_token, "access_token": access_token}), + content_type="application/json") + return response + + def _get_tokens(self, claims): + now = datetime.utcnow() + claims['iat'] = now + claims['nbf'] = now + claims['exp'] = now + timedelta(minutes=self.access_token_timeout) + claims['grant_type'] = 'access_token' algorithm = 'RS256' if self._tls_private_key is not None else 'HS256' encode_key = self._tls_private_key if algorithm == 'RS256' else self._web_secret_key - encoded = jwt.encode(user, encode_key, algorithm=algorithm) + access_token = jwt.encode(claims, encode_key, algorithm=algorithm) + claims['exp'] = now + timedelta(minutes=self.refresh_token_timeout) + claims['grant_type'] = 'refresh_token' + refresh_token = jwt.encode(claims, encode_key, algorithm=algorithm) + return access_token.decode('utf-8'), refresh_token.decode('utf8') - return Response(encoded, content_type="text/plain") + def renew_auth_token(self, env, data): + """ + Creates a new authentication access token to be returned to the caller. The + response will be a text/plain encoded user. Request should contain: + • Content Type: application/json + • Authorization: BEARER + • Body (optional): + { + "current_access_token": "" + } + + :param env: + :param data: + :return: + """ + current_access_token = data.get('current_access_token') + from volttron.platform.web import get_bearer, get_user_claim_from_bearer, NotAuthorized + try: + current_refresh_token = get_bearer(env) + claims = get_user_claim_from_bearer(current_refresh_token, web_secret_key=self._web_secret_key, + tls_public_key=self._tls_public_key) + except NotAuthorized: + _log.error("Unauthorized user attempted to connect to {}".format(env.get('PATH_INFO'))) + return Response('Unauthorized User', status="401 Unauthorized") + + except jwt.ExpiredSignatureError: + _log.error("User attempted to connect to {} with an expired signature".format(env.get('PATH_INFO'))) + return Response('Unauthorized User', status="401 Unauthorized") + + if claims.get('grant_type') != 'refresh_token' or not claims.get('groups'): + return Response('Invalid refresh token.', status="401 Unauthorized") + else: + # TODO: Consider blacklisting and reissuing refresh tokens also when used. + new_access_token, _ = self._get_tokens(claims) + if current_access_token: + pass # TODO: keep current subscriptions? blacklist old token? + return Response(json.dumps({"access_token": new_access_token}), content_type="application/json") + + def revoke_auth_token(self, env, data): + # TODO: Blacklist old token? Immediately close websockets? + return Response('DELETE /authenticate is not yet implemented', status='501 Not Implemented', + content_type='text/plain') def __get_user(self, username, password): """ diff --git a/volttron/platform/web/platform_web_service.py b/volttron/platform/web/platform_web_service.py index 055e2154c6..62a6d6fddd 100644 --- a/volttron/platform/web/platform_web_service.py +++ b/volttron/platform/web/platform_web_service.py @@ -48,6 +48,7 @@ import gevent import gevent.pywsgi +import jwt from cryptography.hazmat.primitives import serialization from gevent import Greenlet from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -186,18 +187,20 @@ def remove_unconnnected_routes(self): def get_user_claims(self, bearer): from volttron.platform.web import get_user_claim_from_bearer if self.core.messagebus == 'rmq': - return get_user_claim_from_bearer(bearer, - tls_public_key=self._certs.get_cert_public_key( - get_fq_identity(self.core.identity))) - if self.web_ssl_cert is not None: - return get_user_claim_from_bearer(bearer, - tls_public_key=CertWrapper.get_cert_public_key(self.web_ssl_cert)) + claims = get_user_claim_from_bearer(bearer, + tls_public_key=self._certs.get_cert_public_key( + get_fq_identity(self.core.identity))) + elif self.web_ssl_cert is not None: + claims = get_user_claim_from_bearer(bearer, + tls_public_key=CertWrapper.get_cert_public_key(self.web_ssl_cert)) elif self._web_secret_key is not None: - return get_user_claim_from_bearer(bearer, web_secret_key=self._web_secret_key) + claims = get_user_claim_from_bearer(bearer, web_secret_key=self._web_secret_key) else: raise ValueError("Configuration error secret key or web ssl cert must be not None.") + return claims if claims.get('grant_type') == 'access_token' else {} + @RPC.export def websocket_send(self, endpoint, message): _log.debug("Sending data to {} with message {}".format(endpoint, @@ -491,11 +494,6 @@ def app_routing(self, env, start_response): if isinstance(retvalue, Response): return retvalue(env, start_response) - #return self.process_response(start_response, retvalue) - elif isinstance(retvalue, Response): # werkzueg Response - for d in retvalue(env, start_response): - print(d) - return retvalue(env, start_response) else: return retvalue[0] @@ -739,6 +737,10 @@ def jsonrpc_verify_and_dispatch(self, authentication): except NotAuthorized: _log.error("Unauthorized user attempted to connect to platform.") return False + except jwt.ExpiredSignatureError: + _log.error("User attempted to connect with an expired signature.") + return False + return True @@ -803,9 +805,11 @@ def startupagent(self, sender, **kwargs): if parsed.scheme == 'https': if self.core.messagebus == 'rmq': ssl_private_key = self._certs.get_pk_bytes(get_fq_identity(self.core.identity)) + ssl_public_key = self._certs.get_cert_public_key(get_fq_identity(self.core.identity)) else: ssl_private_key = CertWrapper.get_private_key(ssl_key) - for rt in AuthenticateEndpoints(tls_private_key=ssl_private_key).get_routes(): + ssl_public_key = CertWrapper.get_cert_public_key(self.web_ssl_cert) + for rt in AuthenticateEndpoints(tls_private_key=ssl_private_key, tls_public_key=ssl_public_key).get_routes(): self.registeredroutes.append(rt) else: # We don't have a private ssl key if we aren't using ssl. diff --git a/volttron/platform/web/templates/login.html b/volttron/platform/web/templates/login.html index 1607c118df..bedb90e5ec 100644 --- a/volttron/platform/web/templates/login.html +++ b/volttron/platform/web/templates/login.html @@ -26,11 +26,9 @@ 'Accept': 'application/json' }, success: function(content){ - alert(content); $("#response").append(content) }, failure: function(content){ - alert("failed!"); $('#response').append("FAILURE "+ content.toString()); }, statusCode: { @@ -39,12 +37,12 @@ }, 200: function(response){ if(location.protocol != 'https:'){ - console.log("HTTP") - document.cookie = "Bearer="+response.responseText+"; path=/"; + console.log("HTTP"); + document.cookie = "Bearer="+response.access_token+"; path=/"; } else{ - console.log("HTTPS") - document.cookie = "Bearer="+response.responseText+"; secure=True; path=/"; + console.log("HTTPS"); + document.cookie = "Bearer="+response.access_token+"; secure=True; path=/"; } window.location=window.location.origin + "/admin/pending_auth_reqs.html"; diff --git a/volttrontesting/platform/web/test_admin.py b/volttrontesting/platform/web/test_admin.py index 5a51722cf6..5706584311 100644 --- a/volttrontesting/platform/web/test_admin.py +++ b/volttrontesting/platform/web/test_admin.py @@ -29,7 +29,7 @@ def test_can_authenticate_admin_user(volttron_instance_web, user_pass): resp = webadmin.authenticate(user, password) assert resp.ok - assert resp.headers.get('Content-Type') == 'text/plain' + assert resp.headers.get('Content-Type') == 'application/json' resp = webadmin.authenticate('fake', password) assert resp.status_code == 401 # unauthorized @@ -49,7 +49,7 @@ def test_can_create_admin_user(volttron_instance_web, user_pass): resp = webadmin.authenticate(user, password) assert resp.ok - assert resp.headers.get('Content-Type') == 'text/plain' + assert resp.headers.get('Content-Type') == 'application/json' resp = webadmin.authenticate('fake', password) assert resp.status_code == 401 # unauthorized diff --git a/volttrontesting/platform/web/test_discovery.py b/volttrontesting/platform/web/test_discovery.py index 1743af774e..8bf452ba44 100644 --- a/volttrontesting/platform/web/test_discovery.py +++ b/volttrontesting/platform/web/test_discovery.py @@ -42,7 +42,7 @@ from volttron.platform.web import DiscoveryInfo from volttrontesting.utils.web_utils import get_test_web_env - +from volttrontesting.fixtures.volttron_platform_fixtures import volttron_instance_web def test_discovery_endpoint(volttron_instance_web): """ diff --git a/volttrontesting/platform/web/test_web_authentication.py b/volttrontesting/platform/web/test_web_authentication.py index d47facc5f2..266094a192 100644 --- a/volttrontesting/platform/web/test_web_authentication.py +++ b/volttrontesting/platform/web/test_web_authentication.py @@ -1,23 +1,38 @@ - +import os +import json from urllib.parse import urlencode +import gevent +from datetime import datetime, timedelta from mock import MagicMock from deepdiff import DeepDiff -import jwt + import pytest +from volttron.platform.web import PlatformWebService +from volttrontesting.utils.utils import AgentMock + +try: + import jwt +except ImportError: + pytest.mark.skip(reason="JWT is missing! Web is not enabled for this installation of VOLTTRON") + +from volttron.platform import is_rabbitmq_available from volttron.platform.agent.known_identities import AUTH -from volttron.platform.certs import CertWrapper +from volttron.platform.certs import CertWrapper, Certs from volttron.platform.vip.agent import Agent from volttron.utils import get_random_key -from volttrontesting.utils.platformwrapper import create_volttron_home -from volttrontesting.utils.utils import AgentMock +from volttrontesting.utils.platformwrapper import create_volttron_home, with_os_environ from volttrontesting.utils.web_utils import get_test_web_env -from volttron.platform.web.platform_web_service import PlatformWebService from volttron.platform.web.admin_endpoints import AdminEndpoints from volttron.platform.web.authenticate_endpoint import AuthenticateEndpoints from volttrontesting.fixtures.cert_fixtures import certs_profile_1 -from volttrontesting.fixtures.volttron_platform_fixtures import get_test_volttron_home +from volttrontesting.fixtures.volttron_platform_fixtures import get_test_volttron_home, volttron_instance_web + +HAS_RMQ = is_rabbitmq_available() +ci_skipif = pytest.mark.skipif(os.getenv('CI', None) == 'true', reason='SSL does not work in CI') +rmq_skipif = pytest.mark.skipif(not HAS_RMQ, + reason='RabbitMQ is not setup and/or SSL does not work in CI') @pytest.mark.parametrize("encryption_type", ("private_key", "tls")) @@ -40,16 +55,116 @@ def test_jwt_encode(encryption_type): assert not DeepDiff(claims, new_claimes) +# Child of AuthenticateEndpoints. +# Exactly the same but includes helper methods to set access and refresh token timeouts +class MockAuthenticateEndpoints(AuthenticateEndpoints): + def set_refresh_token_timeout(self, timeout): + self.refresh_token_timeout = timeout + def set_access_token_timeout(self, timeout): + self.access_token_timeout = timeout + +# Setup test values for authenticate tests +def set_test_admin(): + authorize_ep = MockAuthenticateEndpoints(web_secret_key=get_random_key()) + authorize_ep.set_access_token_timeout(0.1) + authorize_ep.set_refresh_token_timeout(0.2) + AdminEndpoints().add_user("test_admin", "Pass123", groups=['admin']) + test_user = {"username": "test_admin", "password": "Pass123"} + gevent.sleep(1) + return authorize_ep, test_user + +def test_authenticate_get_request_fails(): + with get_test_volttron_home(messagebus='zmq'): + authorize_ep, test_user = set_test_admin() + env = get_test_web_env('/authenticate', method='GET') + response = authorize_ep.handle_authenticate(env, test_user) + assert ('Content-Type', 'text/plain') in response.headers.items() + assert '405 Method Not Allowed' in response.status -def test_authenticate_must_use_post_request(): +def test_authenticate_post_request(): with get_test_volttron_home(messagebus='zmq'): + authorize_ep, test_user = set_test_admin() + env = get_test_web_env('/authenticate', method='POST') + response = authorize_ep.handle_authenticate(env, test_user) + assert ('Content-Type', 'application/json') in response.headers.items() + assert '200 OK' in response.status + response_token = json.loads(response.response[0].decode('utf-8')) + refresh_token = response_token['refresh_token'] + access_token = response_token["access_token"] + assert 3 == len(refresh_token.split('.')) + assert 3 == len(access_token.split(".")) - env = get_test_web_env('/authenticate', method='GET') - authorize_ep = AuthenticateEndpoints(web_secret_key=get_random_key()) - response = authorize_ep.get_auth_token(env, {}) +def test_authenticate_put_request(): + with get_test_volttron_home(messagebus='zmq'): + + authorize_ep, test_user = set_test_admin() + # Get tokens for test + env = get_test_web_env('/authenticate', method='POST') + response = authorize_ep.handle_authenticate(env, test_user) + response_token = json.loads(response.response[0].decode('utf-8')) + refresh_token = response_token['refresh_token'] + access_token = response_token["access_token"] + + # Test PUT Request + env = get_test_web_env('/authenticate', method='PUT') + env["HTTP_AUTHORIZATION"] = "BEARER " + refresh_token + response = authorize_ep.handle_authenticate(env, data={}) + assert ('Content-Type', 'application/json') in response.headers.items() + assert '200 OK' in response.status + + +def test_authenticate_put_request_access_expires(): + with get_test_volttron_home(messagebus='zmq'): + + authorize_ep, test_user = set_test_admin() + # Get tokens for test + env = get_test_web_env('/authenticate', method='POST') + response = authorize_ep.handle_authenticate(env, test_user) + response_token = json.loads(response.response[0].decode('utf-8')) + refresh_token = response_token['refresh_token'] + access_token = response_token["access_token"] + + # Get access token after previous token expires. Verify they are different + gevent.sleep(7) + env = get_test_web_env('/authenticate', method='PUT') + env["HTTP_AUTHORIZATION"] = "BEARER " + refresh_token + response = authorize_ep.handle_authenticate(env, data={}) + assert ('Content-Type', 'application/json') in response.headers.items() + assert '200 OK' in response.status + assert access_token != json.loads(response.response[0].decode('utf-8'))["access_token"] + +def test_authenticate_put_request_refresh_expires(): + with get_test_volttron_home(messagebus='zmq'): + + authorize_ep, test_user = set_test_admin() + # Get tokens for test + env = get_test_web_env('/authenticate', method='POST') + response = authorize_ep.handle_authenticate(env, test_user) + response_token = json.loads(response.response[0].decode('utf-8')) + refresh_token = response_token['refresh_token'] + access_token = response_token["access_token"] + + # Wait for refresh token to expire + gevent.sleep(20) + env = get_test_web_env('/authenticate', method='PUT') + env["HTTP_AUTHORIZATION"] = "BEARER " + refresh_token + response = authorize_ep.handle_authenticate(env, data={}) assert ('Content-Type', 'text/html') in response.headers.items() - assert '401 Unauthorized' in response.status + assert "401 Unauthorized" in response.status + +def test_authenticate_delete_request(): + with get_test_volttron_home(messagebus='zmq'): + authorize_ep, test_user = set_test_admin() + # Get tokens for test + env = get_test_web_env('/authenticate', method='POST') + response = authorize_ep.handle_authenticate(env, test_user) + + # Touch Delete endpoint + env = get_test_web_env('/authenticate', method='DELETE') + response = authorize_ep.handle_authenticate(env, test_user) + assert ('Content-Type', 'text/plain') in response.headers.items() + assert '501 Not Implemented' in response.status def test_no_private_key_or_passphrase(): @@ -67,6 +182,15 @@ def test_both_private_key_and_passphrase(): tls_private_key=certs.server_certs[0].key) +@pytest.fixture() +def mock_platformweb_service(): + PlatformWebService.__bases__ = (AgentMock.imitate(Agent, Agent()),) + platformweb = PlatformWebService(serverkey=MagicMock(), identity=MagicMock(), address=MagicMock(), bind_web_address=MagicMock()) + rpc_caller = platformweb.vip.rpc + platformweb._admin_endpoints = AdminEndpoints(rpc_caller=rpc_caller) + yield platformweb + + @pytest.mark.parametrize("scheme", ("http", "https")) def test_authenticate_endpoint(scheme): kwargs = {} @@ -98,51 +222,105 @@ def test_authenticate_endpoint(scheme): invalid_login_username_params = dict(username='fooey', password=passwd) - response = authorizeep.get_auth_token(env, invalid_login_username_params) + response = authorizeep.get_auth_tokens(env, invalid_login_username_params) - assert '401' == response.status - # TODO: Get the actual response content here # assert '401 Unauthorized' in response.content + assert '401 UNAUTHORIZED' == response.status invalid_login_password_params = dict(username=user, password='hazzah') - response = authorizeep.get_auth_token(env, invalid_login_password_params) + response = authorizeep.get_auth_tokens(env, invalid_login_password_params) - assert '401' == response.status + assert '401 UNAUTHORIZED' == response.status valid_login_params = urlencode(dict(username=user, password=passwd)) - response = authorizeep.get_auth_token(env, valid_login_params) + response = authorizeep.get_auth_tokens(env, valid_login_params) assert '200 OK' == response.status - assert "text/plain" in response.content_type - assert 3 == len(response.response[0].decode('utf-8').split('.')) + assert "application/json" in response.content_type + response_data = json.loads(response.data.decode('utf-8')) + assert 3 == len(response_data["refresh_token"].split('.')) + assert 3 == len(response_data["access_token"].split('.')) -@pytest.fixture() -def mock_platformweb_service(): - PlatformWebService.__bases__ = (AgentMock.imitate(Agent, Agent()),) - platformweb = PlatformWebService(serverkey=MagicMock(), identity=MagicMock(), address=MagicMock(), bind_web_address=MagicMock()) - rpc_caller = platformweb.vip.rpc - platformweb._admin_endpoints = AdminEndpoints(rpc_caller=rpc_caller) - yield platformweb +@pytest.mark.web +def test_get_credentials(volttron_instance_web): + instance = volttron_instance_web + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") + assert len(auth_pending) == 0 + with with_os_environ(instance.env): + pending_agent = Agent(identity="PendingAgent") + task = gevent.spawn(pending_agent.core.run) + task.join(timeout=5) + pending_agent.core.stop() + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") -@pytest.mark.web -def test_get_credentials(mock_platformweb_service): - mock_platformweb_service._admin_endpoints._pending_auths = mock_platformweb_service._admin_endpoints._rpc_caller.call(AUTH, 'get_authorization_pending') - mock_platformweb_service._admin_endpoints._denied_auths = mock_platformweb_service._admin_endpoints._rpc_caller.call(AUTH, 'get_authorization_denied') - pass + assert len(auth_pending) == 1 @pytest.mark.web -def test_accept_credential(mock_platformweb_service): - mock_platformweb_service._admin_endpoints._pending_auths = mock_platformweb_service._admin_endpoints._rpc_caller.call(AUTH, 'get_authorization_pending').get() - mock_platformweb_service._admin_endpoints._denied_auths = mock_platformweb_service._admin_endpoints._rpc_caller.call(AUTH, 'get_authorization_denied').get() - pass +def test_accept_credential(volttron_instance_web): + instance = volttron_instance_web + with with_os_environ(instance.env): + pending_agent = Agent(identity="PendingAgent") + task = gevent.spawn(pending_agent.core.run) + task.join(timeout=5) + pending_agent.core.stop() + + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") + assert len(auth_pending) == 1 + + auth_approved = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_approved").get() + assert len(auth_approved) == 0 + + print(f"agent uuid: {pending_agent.core.agent_uuid}") + instance.dynamic_agent.vip.rpc.call(AUTH, "approve_authorization_failure", auth_pending[0]["user_id"]).get() + auth_approved = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_approved").get() + + assert len(auth_approved) == 1 @pytest.mark.web -def test_deny_credential(): - pass +def test_deny_credential(volttron_instance_web): + instance = volttron_instance_web + with with_os_environ(instance.env): + pending_agent = Agent(identity="PendingAgent") + task = gevent.spawn(pending_agent.core.run) + task.join(timeout=5) + pending_agent.core.stop() + + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") + assert len(auth_pending) == 1 + + auth_denied = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_denied").get() + assert len(auth_denied) == 0 + + print(f"agent uuid: {pending_agent.core.agent_uuid}") + instance.dynamic_agent.vip.rpc.call(AUTH, "deny_authorization_failure", auth_pending[0]["user_id"]).get() + auth_denied = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_denied").get() + + assert len(auth_denied) == 1 @pytest.mark.web -def test_delete_credential(): - pass +def test_delete_credential(volttron_instance_web): + instance = volttron_instance_web + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") + assert len(auth_pending) == 0 + with with_os_environ(instance.env): + pending_agent = Agent(identity="PendingAgent") + task = gevent.spawn(pending_agent.core.run) + task.join(timeout=5) + pending_agent.core.stop() + + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + print(f"Auth pending is: {auth_pending}") + assert len(auth_pending) == 1 + + instance.dynamic_agent.vip.rpc.call(AUTH, "delete_authorization_failure", auth_pending[0]["user_id"]).get() + auth_pending = instance.dynamic_agent.vip.rpc.call(AUTH, "get_authorization_pending").get() + + assert len(auth_pending) == 0