Merge branch 'master' into wordcloud_generator

.github/workflows/black.yaml

@@ -36,6 +36,11 @@ jobs:
         uses: psf/black@stable
         with:
             src: "word_cloud_generator"
+
+      - name: JSM Metric Collection
+        uses: psf/black@stable
+        with:
+            src: "jsm_metric_collection"
 
       - name: Openstack-Rally-Tester
         uses: psf/black@stable

.github/workflows/jsm_metric_collection.yaml

@@ -0,0 +1,33 @@
+name: DNS Entry Checker Unittest
+
+on: 
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "jsm_metric_collection/**"
+      - ".github/workflows/jsm_metric_collection.yaml"
+
+jobs:
+  test_with_unit_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r jsm_metric_collection/requirements.txt
+      - name: Test with unittest
+        run: |
+            cd jsm_metric_collection
+            python3 -m unittest discover -s ./test -p "test_*.py"

.github/workflows/pynetbox_codecov.yaml

@@ -0,0 +1,40 @@
+name: Pynetbox Codecov
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "pynetbox_data_uploader/**"
+      - ".github/workflows/pynetbox.yaml"
+
+jobs:
+  Codecov:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ["3.x"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd pynetbox_query
+          pip install -r requirements.txt
+          
+
+      - name: Run tests and collect coverage
+        run: cd pynetbox_query && python3 -m pytest --cov-report xml:coverage.xml --cov
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+          files: ./pynetbox_query/coverage.xml

.github/workflows/pynetbox_pylint.yaml

@@ -0,0 +1,35 @@
+name: Pynetbox Pylint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "pynetbox_data_uploader/**"
+      - ".github/workflows/pynetbox.yaml"
+
+jobs:
+  Pylint:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ["3.x"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd pynetbox_query
+          pip install -r requirements.txt
+          
+          
+
+      - name: Analyse with pylint
+        run: cd pynetbox_query && pylint $(git ls-files '*.py')

.github/workflows/pynetbox_tests.yaml

@@ -0,0 +1,34 @@
+name: Pynetbox Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "pynetbox_data_uploader/**"
+      - ".github/workflows/pynetbox.yaml"
+
+jobs:
+  Test:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ["3.x"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd pynetbox_query
+          pip install -r requirements.txt
+          
+
+      - name: Run tests and collect coverage
+        run: cd pynetbox_query && python3 -m pytest

OpenStack-Rabbit-Consumer/rabbit_consumer/message_consumer.py

@@ -27,6 +27,10 @@ def is_aq_managed_image(vm_data: VmData) -> bool:
     is for an Aquilon VM.
     """
     image = openstack_api.get_image(vm_data)
+    if not image:
+        logger.info("No image found for %s", vm_data.virtual_machine_id)
+        return False
+
     if "AQ_OS" not in image.metadata:
         logger.debug("Skipping non-Aquilon image: %s", image.name)
         return False

OpenStack-Rabbit-Consumer/rabbit_consumer/openstack_api.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import List, Optional
 
 import openstack
 from openstack.compute.v2.image import Image
@@ -79,12 +79,15 @@ def get_server_metadata(vm_data: VmData) -> dict:
     return server.metadata
 
 
-def get_image(vm_data: VmData) -> Image:
+def get_image(vm_data: VmData) -> Optional[Image]:
     """
     Gets the image name from Openstack for the virtual machine.
     """
     server = get_server_details(vm_data)
     uuid = server.image.id
+    if not uuid:
+        return None
+
     with OpenstackConnection() as conn:
         image = conn.compute.find_image(uuid)
         return image

OpenStack-Rabbit-Consumer/test/test_message_consumer.py

@@ -332,6 +332,17 @@ def test_is_aq_managed_image(openstack_api, vm_data):
     openstack_api.get_image.assert_called_once_with(vm_data)
 
 
+@patch("rabbit_consumer.message_consumer.openstack_api")
+def test_is_aq_managed_image_missing_image(openstack_api, vm_data):
+    """
+    Test that the function returns False when the image is not AQ managed
+    """
+    openstack_api.get_image.return_value = None
+
+    assert not is_aq_managed_image(vm_data)
+    openstack_api.get_image.assert_called_once_with(vm_data)
+
+
 @patch("rabbit_consumer.message_consumer.VmData")
 @patch("rabbit_consumer.message_consumer.openstack_api")
 def test_is_aq_managed_image_missing_key(openstack_api, vm_data):

OpenStack-Rabbit-Consumer/test/test_openstack_api.py

@@ -9,6 +9,7 @@
     check_machine_exists,
     get_server_details,
     get_server_networks,
+    get_image,
 )
 
 
@@ -122,3 +123,16 @@ def test_get_server_networks_no_internal(server_details, vm_data):
 
     result = get_server_networks(vm_data)
     assert not result
+
+
+@patch("rabbit_consumer.openstack_api.get_server_details")
+def test_get_image_no_image_id(server_details, vm_data):
+    """
+    Tests that get image handles an empty image UUID
+    usually when a volume was used instead of an image
+    """
+    server_details.return_value = NonCallableMock()
+    server_details.return_value.image.id = None
+
+    result = get_image(vm_data)
+    assert not result

OpenStack-Rabbit-Consumer/version.txt

@@ -1 +1 @@
-2.3.4
+2.3.5

OpenStack-accounting/etc/thecount/thecount.conf.example

@@ -0,0 +1,2 @@
+[database]
+connection =mysql+pymysql://<dbusername>:<dbpassword>@<dbhost>:3306

OpenStack-accounting/readme.md

@@ -0,0 +1,23 @@
+# TheCount
+
+
+## Prerequisites
+
+The following packages are required:
+ - python3-requests
+ - python36-sqlalchemy
+ - python36-PyMySQL
+
+A database user with read access to the relevant databases for each OpenStack component you are accounting for
+
+## Installation
+
+Copy the scripts into `/usr/local/sbin` as shown in this repo
+Create a config file with a database connection string in the format shown in `/etc/thecount/thecount.conf.example` in `/etc/thecount/thecount.conf`
+Create the stored procedures in the `sql` directory appropriate db.
+
+## Use
+
+`now-accounting.sh` generates accounting for the last 24 hours - recommend setting up a cron to run this at midnight
+`past-accounting.sh` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for past usage where possible
+`*-extract_accounting.py` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for that component

OpenStack-accounting/requirements.txt

@@ -0,0 +1,3 @@
+python3-requests
+python36-sqlalchemy
+python36-PyMySQL

OpenStack-accounting/sql/cinder_get_accounting_data.sql

@@ -0,0 +1,44 @@
+CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
+BEGIN
+/*
+   This procedure generates accounting data for cinder
+*/
+SELECT
+    IFNULL(v.availability_zone, 'nova') AS AvailabilityZone,
+    p.name AS Project,
+    pp.name AS Department,
+    COUNT(v.id) AS Volumes,
+    "Volume" as CinderType,
+    @VolumeSeconds:=SUM(IF(v.created_at <= starttime /* Captures Volumes which were created outside of the period deleted out of the period */
+            AND (v.deleted_at >= endtime
+            OR ISNULL(v.deleted_at)),
+        TIMESTAMPDIFF(SECOND,
+            starttime,
+            endtime),
+        IF(v.created_at <= starttime /* Captures Volumes which were created before the period and deleted during the period */
+                AND v.deleted_at < endtime,
+            TIMESTAMPDIFF(SECOND,
+                starttime,
+                v.deleted_at),
+            IF(v.created_at > starttime /* Captures Volumes which were created during the period and deleted outside the period */
+                    AND (v.deleted_at >= endtime
+                    OR ISNULL(v.deleted_at)),
+                TIMESTAMPDIFF(SECOND,
+                    v.created_at,
+                    endtime),
+                TIMESTAMPDIFF(SECOND,
+                    v.created_at,
+                    v.deleted_at))))) AS Volume_Seconds, /* Generates a count of seconds Volumes were running */
+    v.size AS Volume_GB
+FROM
+    cinder.volumes v
+        JOIN
+    keystone.project p ON v.project_id = p.id
+        JOIN
+    keystone.project pp ON p.parent_id = pp.id
+WHERE
+    v.created_at <= endtime
+        AND (v.deleted_at >= starttime
+        OR ISNULL(v.deleted_at))
+GROUP BY v.availability_zone , v.size , p.name , pp.name;
+END

OpenStack-accounting/sql/glance_get_accounting_data.sql

@@ -0,0 +1,48 @@
+CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
+BEGIN
+/*
+   This procedure generates accounting data for glance
+*/
+SELECT
+    p.name AS Project,
+    pp.name AS Department,
+    COUNT(g.id) AS Images,
+    ip.value as GlanceType,
+    if(il.value like "%rbd%", "RBD" ,if(il.value like "%swift%","OBJECT","UNKNONWN")) as StorageBackend,
+    @ImageSeconds:=SUM(IF(g.created_at <= starttime /* Captures Images which were created outside of the period deleted out of the period */
+            AND (g.deleted_at >= endtime
+            OR ISNULL(g.deleted_at)),
+        TIMESTAMPDIFF(SECOND,
+            starttime,
+            endtime),
+        IF(g.created_at <= starttime /* Captures Images which were created before the period and deleted during the period */
+                AND g.deleted_at < endtime,
+            TIMESTAMPDIFF(SECOND,
+                starttime,
+                g.deleted_at),
+            IF(g.created_at > starttime /* Captures Images which were created during the period and deleted outside the period */
+                    AND (g.deleted_at >= endtime
+                    OR ISNULL(g.deleted_at)),
+                TIMESTAMPDIFF(SECOND,
+                    g.created_at,
+                    endtime),
+                TIMESTAMPDIFF(SECOND,
+                    g.created_at,
+                    g.deleted_at))))) AS Image_Seconds, /* Generates a count of seconds Images were running */
+    g.size/(1024 * 1024 * 1024) AS Glance_GB
+FROM
+    glance.images g
+    join
+    glance.image_properties ip on g.id = ip.image_id and ip.name = "image_type"
+    join
+    glance.image_locations il on g.id = il.image_id
+        JOIN
+    keystone.project p ON g.owner = p.id
+        JOIN
+    keystone.project pp ON p.parent_id = pp.id
+WHERE
+    g.created_at <= endtime
+        AND (g.deleted_at >= starttime
+        OR g.deleted_at is null )
+GROUP BY ip.value, g.size , p.name , pp.name,if(il.value like "%rbd%", "SIRIUS" ,if(il.value like "%swift%","ECHO","UNKNONWN"));
+END