Merge branch 'develop' into add-timescaledb-tests

.gitignore

@@ -28,3 +28,8 @@ __pycache__
 start-rabbitmq
 stop-rabbitmq
 rabbitmq.log
+*.result.txt
+.coverage
+htmlcov/
+MnesiaCore.*
+rabbitmq-server.download.tar.xz

README.md

@@ -1,4 +1,5 @@
 ![image](docs/source/images/VOLLTRON_Logo_Black_Horizontal_with_Tagline.png)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/fcf58045b4804edf8f4d3ecde3016f76)](https://app.codacy.com/gh/VOLTTRON/volttron?utm_source=github.com&utm_medium=referral&utm_content=VOLTTRON/volttron&utm_campaign=Badge_Grade_Settings)
 
 VOLTTRON™ is an open source platform for distributed sensing and control. The
 platform provides services for collecting and storing data from buildings and
@@ -273,10 +274,10 @@ tail volttron.log
 Listener agent heartbeat publishes appear in the logs as:
 
 ```sh
-2016-10-17 18:17:52,245 (listeneragent-3.2 11367) listener.agent INFO: Peer: 'pubsub', Sender: 'listeneragent-3.2_1'
-:, Bus: u'', Topic: 'heartbeat/listeneragent-3.2_1', Headers:
-{'Date': '2016-10-18T01:17:52.239724+00:00', 'max_compatible_version': u'', 'min_compatible_version': '3.0'},
-Message: {'status': 'GOOD', 'last_updated': '2016-10-18T01:17:47.232972+00:00', 'context': 'hello'}
+2020-04-20 18:49:31,395 (listeneragent-3.3 13458) __main__ INFO: Peer: pubsub, Sender: listeneragent-3.2_1:, Bus: , Topic: heartbeat/listeneragent-3.2_1, Headers: {'TimeStamp': '2020-04-20T18:49:31.393651+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''}, Message:
+'GOOD'
+2020-04-20 18:49:36,394 (listeneragent-3.3 13458) __main__ INFO: Peer: pubsub, Sender: listeneragent-3.2_1:, Bus: , Topic: heartbeat/listeneragent-3.2_1, Headers: {'TimeStamp': '2020-04-20T18:49:36.392294+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''}, Message:
+'GOOD'
 ```
 
 To top the platform run the following command:

bootstrap.py

@@ -122,8 +122,9 @@ def pip(operation, args, verbose=None, upgrade=False, offline=False):
     subprocess.check_call(cmd)
 
 
-def update(operation, verbose=None, upgrade=False, offline=False, optional_requirements=[]):
+def update(operation, verbose=None, upgrade=False, offline=False, optional_requirements=[], rabbitmq_path=None):
     """Install dependencies in setup.py and requirements.txt."""
+    print("UPDATE: {}".format(optional_requirements))
     assert operation in ['install', 'wheel']
     wheeling = operation == 'wheel'
     path = os.path.dirname(__file__) or '.'
@@ -153,14 +154,15 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi
 
     try:
         # Install rmq server if needed
-        if 'rabbitmq' in optional_requirements:
-            install_rabbit(default_rmq_dir)
+        if rabbitmq_path:
+            install_rabbit(rabbitmq_path)
     except Exception as exc:
         _log.error("Error installing RabbitMQ package {}".format(traceback.format_exc()))
 
 
 def install_rabbit(rmq_install_dir):
-
+    # Install gevent friendly pika
+    pip('install', ['gevent-pika==0.3'], False, True, offline=False)
     # try:
     process = subprocess.Popen(["which", "erl"], stderr=subprocess.PIPE,  stdout=subprocess.PIPE)
     (output, error) = process.communicate()
@@ -281,7 +283,18 @@ def main(argv=sys.argv):
     po = parser.add_argument_group('Extra packaging options')
     for arg in extras_require:
         po.add_argument('--'+arg, action='append_const', const=arg, dest="optional_args")
+
     # Add rmq download actions.
+    rabbitmq = parser.add_argument_group('rabbitmq options')
+    rabbitmq.add_argument(
+        '--rabbitmq', action='store', const=default_rmq_dir,
+        nargs='?',
+        help='install rabbitmq server and its dependencies. '
+             'optional argument: Install directory '
+             'that exists and is writeable. RabbitMQ server '
+             'will be installed in a subdirectory.'
+             'Defaults to ' + default_rmq_dir)
+
     #optional_args = []
     # if os.path.exists('optional_requirements.json'):
     #     po = parser.add_argument_group('Extra packaging options')
@@ -337,7 +350,7 @@ def main(argv=sys.argv):
     if sys.base_prefix != sys.prefix:
         # The script was called from a virtual environment Python, so update
         update(options.operation, options.verbose,
-               options.upgrade, options.offline, options.optional_args)
+               options.upgrade, options.offline, options.optional_args, options.rabbitmq)
     else:
         # The script was called from the system Python, so bootstrap
         try:
@@ -364,6 +377,9 @@ def main(argv=sys.argv):
         if options.verbose is not None:
             args.append('--verbose' if options.verbose else '--quiet')
 
+        if options.rabbitmq is not None:
+            args.append('--rabbitmq={}'.format(options.rabbitmq))
+
         # Transfer dynamic properties to the subprocess call 'update'.
         # Clip off the first two characters expecting long parameter form.
         for arg in options.optional_args:

ci-integration/run-test-docker.sh

@@ -70,7 +70,8 @@ run_test(){
     base_filename="$(basename "$filename")"
     # Start the docker run module.
     docker run -e "IGNORE_ENV_CHECK=1" -e "CI=$CI" --name "$base_filename" \
-        -t volttron_test_image pytest "$filename" > "$base_filename.result.txt" 2>&1 &
+            -t --network="host" -v /var/run/docker.sock:/var/run/docker.sock volttron_test_image \
+            pytest "$filename" > "$base_filename.result.txt" 2>&1 &
     runningprocs+=($!)
     outputfiles+=("$base_filename.result.txt")
     containernames+=("$base_filename")
@@ -113,9 +114,11 @@ process_pid(){
                 if [[ ${FAST_FAIL} -eq 0 && -n ${CI} ]]; then
                     docker logs "${containernames[$index]}"
                 fi
-                if [[ ${FAST_FAIL} -eq 0 ]]; then
+                if [ ${FAST_FAIL} ]; then
                     echo "Exiting cleanly now!"
                     exit_cleanly
+                else
+                    echo "Test failed. Keep running rest of tests."
                 fi
             fi
         else

ci-integration/virtualization/Dockerfile.testing

@@ -11,4 +11,23 @@ RUN echo "export VOLTTRON_ROOT=${VOLTTRON_ROOT}" > /home/volttron/.bashrc && \
 
 USER root
 WORKDIR ${VOLTTRON_ROOT}
+
+RUN apt-get update
+RUN apt-get install -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg-agent \
+    software-properties-common \
+    apt-utils
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+RUN add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable"
+RUN apt-get update
+RUN apt-get install docker-ce docker-ce-cli containerd.io -y
+
+RUN usermod -aG docker $VOLTTRON_USER
+
 ENTRYPOINT ["/startup/entrypoint.sh"]

ci-integration/virtualization/core/entrypoint.sh

@@ -45,6 +45,15 @@ fi
 # #   exit 1;
 # # fi
 
+# For tests that need to use Docker, we need to restart the virtual machine so that user added to 'docker' group, such as user 'volttron' can have privileges to run Docker.
+# See step 3 of "Manage Docker as a non-root user" https://docs.docker.com/engine/install/linux-postinstall/
+# However, rebooting a container results in a "System has not been booted with systemd as init system (PID 1)."
+# Thus, in order to give Docker privileges to 'volttron' user, '/var/run/docker/sock' is modified to be readable, writable, and executable by all users.
+# This is obviously a security risk, however, given that the container that uses this `entrypoint.sh` is ephemeral and used only for testing, the risk
+# is minimal and more importantly allows successful level one integration testing on Travis CI.
+chmod 777 /var/run/docker.sock
+
+
 if [[ $# -lt 1 ]]; then
   echo "Please provide a command to run (e.g. /bin/bash, volttron -vv)";
   exit 1;

ci-integration/virtualization/requirements_test.txt

@@ -1,7 +1,13 @@
 more-itertools>=4.0.0
-pytest==3.6.4
+pytest==5.4.3
 pytest-timeout
 mock
 websocket-client
 numpy>1.13<2
-pandas
+pandas
+watchdog==0.10.2
+watchdog-gevent==0.1.1
+cryptography==2.3
+docker
+psycopg2
+mysql-connector-python-rf

docs/source/agent-framework/agents-overview.rst

@@ -0,0 +1,43 @@
+.. _Agent-Framework:
+
+===============
+Agents Overview
+===============
+
+Agents in VOLTTRON can be loosely defined as software modules communicating on the platform which perform some function
+on behalf of the user.  Agents may perform a huge variety of tasks, but common use cases involve data collection,
+control of ICS and IOT devices, and various platform management tasks.  Agents implemented using the VOLTTRON agent
+framework inherit a number of capabilities, including message bus connectivity and agent lifecycle.
+
+Agents deployed on VOLTTRON can perform one or more roles which can be broadly classified into the following groups:
+
+-  Platform Agents: Agents which are part of the platform and provide a service to other agents. Examples are the
+   Actuator and Master Driver agents which serve as interfaces between control agents and drivers.
+-  Control Agents: These agents implement algorithms to control the devices of interest and interact with other
+   resources to achieve some goal.
+-  Service Agents: These agents perform various data collection or platform management services.  Agents in this
+   category include weather service agents which collect weather data from remote sources or operations agents which
+   help users maintain situational awareness of their deployment.
+-  Cloud Agents: These agents represent a remote application which needs access to the messages and data on the
+   platform. This agent would subscribe to topics of interest to the remote application and would also allow it publish
+   data to the platform.
+
+The platform includes some valuable services which can be leveraged by agents:
+
+-  Message Bus: All agents and services publish and subscribe to topics on the message bus. This provides a single
+   interface that abstracts the details of devices and agents from each other. Components in the platform basically
+   produce and consume events.
+-  Configuration Store: Using the configuration store, agent operations can be altered ad-hoc without significant
+   disruption or downtime.
+-  Historian Framework: Historian agents automatically collect data from a subset of topics on the message bus and store
+   them in a data store of choice.  Currently SQL, MongoDB, CrateDB and other historians exist, and more can be
+   developed to fit the needs of a deployment by inheriting from the base historian.  The base historian has been
+   developed to be fast and reliable, and to handle many common pitfalls of data collection over a network.
+-  Weather Information: These agents periodically retrieve data from the a remote weather API then format the
+   response and publish it to the platform message bus on a weather topic.
+-  Device interfaces: Drivers publish device data onto the message bus and send control signals issued from control
+   agents to the corresponding device.  Drivers are capable of handling the locking of devices to prevent multiple
+   conflicting directives.
+-  Application Scheduling: This service allows the scheduling of agents’ access to devices in order to prevent conflicts.
+-  Logging service: Agents can publish arbitrary strings to a logging topic and this service will push them to a
+   historian for later analysis.

docs/source/core_services/control/Agent-Execution-Environment.rst → docs/source/agent-framework/aip.rst

@@ -1,3 +1,10 @@
+.. _Agent-Instantiation-and-Packaging:
+
+=======================================
+AIP - Agent Instantiation and Packaging
+=======================================
+
+
 .. _Agent-Execution-Environment:
 
 Used Environmental Variables
@@ -6,3 +13,6 @@ Used Environmental Variables
 - **AGENT_VIP_IDENTITY** - The router address an agent will attempt to connect to.
 - **AGENT_CONFIG** - The path to a configuration file to use during agent launch.
 - **VOLTTRON_HOME** - The home directory where the volttron instances is located.
+
+Documentation coming soon!
+