merged staging

.coveragerc

@@ -0,0 +1,18 @@
+[run]
+omit =
+    simple_vm_client/VirtualMachineService.py
+    simple_vm_client/constants.py
+    simple_vm_client/test_openstack_connector.py
+    simple_vm_client/ttypes.py
+    simple_vm_client/forc_connector/template/test_templates.py
+    simple_vm_client/util/logger.py
+    simple_vm_client/forc_connector/test_forc_connector.py
+    simple_vm_client/VirtualMachineServer.py
+
+    check_env.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:

.github/workflows/codeql-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         queries: +security-extended, security-and-quality
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/coverage.yml

@@ -1,20 +1,35 @@
-name: 'coverage'
+
+# This workflow will install dependencies, create coverage tests and run Pytest Coverage Comment
+# For more information see: https://github.com/MishaKav/pytest-coverage-comment/
+name: pytest-coverage-comment
 on:
-    pull_request:
-    workflow_dispatch:
+  pull_request:
+    branches:
+      - '*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
 
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
 
-jobs:
-    coverage:
-        runs-on: ubuntu-latest
-        steps:
-          - uses: actions/checkout@v4.1.1
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest pytest-cov
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Build coverage file
+        run: |
+          pytest --junitxml=pytest.xml | tee pytest-coverage.txt
 
-          - name: Run Coverage
-            run: cd simplevm-client/simple_vm_client && coverage xml coverage.xml
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml
 
-          - name: Get Cover
-            uses: orgoro/coverage@v3.1
-            with:
-                coverageFile: coverage.xml
-                token: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -9,17 +9,19 @@ repos:
       - id: end-of-file-fixer
       - id: check-yaml
 
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+
+    rev: 23.12.0
     hooks:
       - id: black
+        language_version: python3.11
   - repo: https://github.com/sondrelg/pep585-upgrade
     rev: 'v1.0.1'  # Use the sha / tag you want to point at
     hooks:
       - id: upgrade-type-hints
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
 
@@ -33,7 +35,7 @@ repos:
           - --remove-all-unused-imports
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
       - id: flake8
         args: [ "--config=setup.cfg" ]

Dockerfile

@@ -1,11 +1,24 @@
 FROM python:3.11.4-buster
-RUN  apt-get update -y
-RUN apt-get install -y build-essential
+
+RUN apt-get update -y \
+    && apt-get install -y build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /code
-ADD requirements.txt /code
+
+# Copy requirements and install them first to leverage Docker cache
+COPY requirements.txt /code
 RUN pip install -r requirements.txt
-ADD requirements.yml /code
-ADD ansible.cfg /etc/ansible/
+
+COPY requirements.yml /code
+COPY ansible.cfg /etc/ansible/
 RUN ansible-galaxy install -r requirements.yml
-ADD . /code
+
+# Copy the entire project
+COPY . /code
+
+# Set PYTHONPATH to include the project root
+ENV PYTHONPATH /code
+
 WORKDIR /code/simple_vm_client

Makefile

@@ -18,7 +18,6 @@ thrift_py: ## Builds python code from thrift file
 	thrift --gen py portal_client.thrift
 	cp -a gen-py/VirtualMachineService/. simple_vm_client
 	rm -rf gen-py
-	@echo Remember to fix the imports: for pip relative imports are needed, for others absolute imports
 
 dev-build: ## Build and Start the docker-compose.dev.yml
 	docker-compose -f docker-compose.dev.yml up --build

docker-compose.bibigrid.yml

@@ -41,7 +41,7 @@ services:
 
   # filebeat
   simplevm_filebeat:
-    image: docker.elastic.co/beats/filebeat:8.11.2
+    image: docker.elastic.co/beats/filebeat:8.11.3
     env_file:
       - .env
     volumes:

docker-compose.yml

@@ -27,7 +27,7 @@ services:
   # filebeat
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.11.2
+    image: docker.elastic.co/beats/filebeat:8.11.3
     env_file:
       - .env
     volumes:

pytest.ini

@@ -0,0 +1,6 @@
+# pytest.ini
+
+[pytest]
+addopts = --cov=. --cov-config=.coveragerc
+
+# Add other configuration options as needed

requirements.txt

@@ -1,18 +1,18 @@
-setuptools==69.0.2
+setuptools==69.0.3
 thrift==0.16.0
 python-keystoneclient==5.2.0
-openstacksdk==1.5.0
+openstacksdk==2.0.0
 deprecated==1.2.14
 Click==8.1.7
 ansible==5.1.0
-flake8==6.1.0
-paramiko==2.12.0
+flake8==7.0.0
+paramiko==3.4.0
 ruamel.yaml==0.18.5
 pyvim==3.0.3
 redis==5.0.1
 requests==2.31.0
 pyyaml==6.0.1
-pre-commit==3.5.0
+pre-commit==3.6.0
 types-PyYAML==6.0.12.12
 sympy==1.12
-types-redis==4.6.0.11
+types-redis==4.6.0.20240106

setup.cfg

@@ -27,5 +27,3 @@ warn_unused_configs = True
 [coverage:run]
 include = simple_vm_client/*
 omit = *migrations*, *tests*
-plugins =
-    django_coverage_plugin

simple_vm_client/VirtualMachineHandler.py

@@ -5,29 +5,27 @@
 """
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
-from bibigrid_connector.bibigrid_connector import BibigridConnector
-from forc_connector.forc_connector import ForcConnector
-from openstack_connector.openstack_connector import OpenStackConnector
-from util import thrift_converter
-from util.logger import setup_custom_logger
-from VirtualMachineService import Iface
-
-if TYPE_CHECKING:
-    from ttypes import (
-        VM,
-        Backend,
-        ClusterInfo,
-        ClusterInstance,
-        CondaPackage,
-        Flavor,
-        Image,
-        PlaybookResult,
-        ResearchEnvironmentTemplate,
-        Snapshot,
-        Volume,
-    )
+
+from simple_vm_client.bibigrid_connector.bibigrid_connector import BibigridConnector
+from simple_vm_client.forc_connector.forc_connector import ForcConnector
+from simple_vm_client.openstack_connector.openstack_connector import OpenStackConnector
+from simple_vm_client.util import thrift_converter
+from simple_vm_client.util.logger import setup_custom_logger
+
+from .ttypes import (
+    VM,
+    Backend,
+    ClusterInfo,
+    ClusterInstance,
+    CondaPackage,
+    Flavor,
+    Image,
+    PlaybookResult,
+    ResearchEnvironmentTemplate,
+    Snapshot,
+    Volume,
+)
+from .VirtualMachineService import Iface
 
 logger = setup_custom_logger(__name__)
 
@@ -143,10 +141,13 @@ def get_server(self, openstack_id: str) -> VM:
         return server
 
     def get_servers(self) -> list[VM]:
-        serv = thrift_converter.os_to_thrift_servers(
-            openstack_servers=self.openstack_connector.get_servers()
-        )
-        return serv
+        servers = openstack_servers = self.openstack_connector.get_servers()
+        servers_full = []
+
+        for server in servers:
+            servers_full.append(self.forc_connector.get_playbook_status(server=server))
+        serv = thrift_converter.os_to_thrift_servers(openstack_servers=servers)
+        return servers_full
 
     def get_servers_by_ids(self, server_ids: list[str]) -> list[VM]:
         return thrift_converter.os_to_thrift_servers(

simple_vm_client/VirtualMachineServer.py

@@ -1,4 +1,3 @@
-import os
 import signal
 import ssl
 import sys
@@ -8,8 +7,9 @@
 from thrift.protocol import TBinaryProtocol
 from thrift.server import TServer
 from thrift.transport import TSocket, TSSLSocket, TTransport
-from VirtualMachineHandler import VirtualMachineHandler
-from VirtualMachineService import Processor
+
+from simple_vm_client.VirtualMachineHandler import VirtualMachineHandler
+from simple_vm_client.VirtualMachineService import Processor
 
 USERNAME = "OS_USERNAME"
 PASSWORD = "OS_PASSWORD"
@@ -72,15 +72,5 @@ def catch_shutdown(signal: int, frame: object) -> None:
     server.serve()
 
 
-def check_environment_variables(envs: list[str]) -> None:
-    def check_env(var: str) -> None:
-        if var not in os.environ:
-            click.echo(f"ERROR: There is no {var} set in environment.")
-            click.echo("Please make sure you have sourced your OpenStack rc file")
-            sys.exit()
-
-    list(map(lambda var: check_env(var), envs))
-
-
 if __name__ == "__main__":
     startServer()