Merge branch 'develop' into debugger_documentation

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# Migrate python code style to Black
+
+2d2de2d1e5167b54b253fa496d91e9a3b0b859c1
+e7d4c5be7400bbe6fa5f7b59d34d07a51d15bd31
+e03e94b5439bf30d0b03430980745d5adc9cc631

.github/workflows/ci.yml

@@ -35,12 +35,12 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: build_api
-          path: services/coverage.xml
+          path: services/cov.xml
 
-      - name: Find coverage.xml
+      - name: Find cov.xml
         shell: bash
         run: |
-          find "$GITHUB_WORKSPACE/services" -name "coverage.xml"
+          find "$GITHUB_WORKSPACE/services" -name "cov.xml"
 
   webapp:
     runs-on: ubuntu-latest
@@ -89,28 +89,31 @@ jobs:
         continue-on-error: true
         with:
           name: build_api
-          path: services/coverage.xml
+          path: services/cov.xml
 
       - name: Download Code Coverage Results
         uses: actions/download-artifact@v3
         with:
           name: webapp
           path: webapp
 
-      - name: Find coverage.xml
+      - name: Find cov.xml
         shell: bash
         run: |
-          find "$GITHUB_WORKSPACE" -name "coverage.xml"
+          find "$GITHUB_WORKSPACE" -name "cov.xml"
 
       - name: Find lcov.info
         shell: bash
         run: |
           find "$GITHUB_WORKSPACE" -name "lcov.info"
 
-      - name: Setup SonarScanner
-        uses: warchant/setup-sonar-scanner@v4
+      - uses: actions/setup-java@v3
         with:
-          version: 4.8.0.2856
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup SonarScanner
+        uses: warchant/setup-sonar-scanner@v7
 
       - name: Generate Sonar Properties File
         run: |
@@ -124,10 +127,10 @@ jobs:
           sonar.projectBaseDir=$GITHUB_WORKSPACE
           sonar.projectKey=usdot-jpo-ode_jpo-cvmanager
           sonar.projectName=jpo-cvmanager
-          sonar.python.coverage.reportPaths=$GITHUB_WORKSPACE/services/coverage.xml
-          sonar.python.version=3.9
+          sonar.python.coverage.reportPaths=$GITHUB_WORKSPACE/services/cov.xml
+          sonar.python.version=3.12
           api.sonar.projectBaseDir=$GITHUB_WORKSPACE/services
-          api.sonar.sources=addons/images/bsm_query,addons/images/count_metric,addons/images/iss_health_check,addons/images/rsu_ping,api/src,common/pgquery.py
+          api.sonar.sources=addons/images/bsm_query,addons/images/count_metric,addons/images/firmware_manager,addons/images/iss_health_check,addons/images/rsu_ping,api/src,common/pgquery.py
           api.sonar.tests=addons/tests,api/tests,common/tests
           webapp.sonar.projectBaseDir=$GITHUB_WORKSPACE/webapp
           webapp.sonar.sources=src

.gitignore

@@ -1,6 +1,6 @@
 *.pyc
 *.pyc.*
-.env
+.env*
 .coverage
 cov.xml
 .venv

.vscode/extensions.json

@@ -1,5 +1,3 @@
 {
-    "recommendations": [
-        "esbenp.prettier-vscode"
-    ]
-}
+  "recommendations": ["esbenp.prettier-vscode", "ms-python.black-formatter"]
+}

.vscode/settings.json

@@ -18,5 +18,9 @@
   "python.envFile": "${workspaceFolder}/.env",
   "terminal.integrated.env.windows": {
     "PYTHONPATH": "${workspaceFolder}/services;${workspaceFolder}/services/addons/images/bsm_query;${workspaceFolder}/services/addons/images/count_metric;${workspaceFolder}/services/addons/images/firmware_manager;${workspaceFolder}/services/addons/images/iss_health_check;${workspaceFolder}/services/addons/images/rsu_ping_fetch;${workspaceFolder}/services/api/src;${workspaceFolder}/services/common"
+  },
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": true
   }
 }

.vscode/tasks.json

@@ -21,7 +21,7 @@
         "up": {
           "detached": true,
           "build": true,
-          "services": ["cvmanager_keycloak", "cvmanager_postgres", "firmware_manager"]
+          "services": ["cvmanager_keycloak", "cvmanager_postgres"]
         },
         "files": ["${workspaceFolder}/docker-compose.yml"],
         "envFile": "${workspaceFolder}/.env"

README.md

@@ -150,7 +150,7 @@ For the "Debug Solution" to run properly on Windows 10/11 using WSL, the followi
 - BSM_DB_NAME: The database name for BSM visualization data.
 - SSM_DB_NAME: The database name for SSM visualization data.
 - SRM_DB_NAME: The database name for SRM visualization data.
-- RSU_REST_ENDPOINT: HTTPS endpoint of the deployed RSU REST API in GCP Kubernetes.
+- FIRMWARE_MANAGER_ENDPOINT: Endpoint for the firmware manager deployment's API.
 - CSM_EMAIL_TO_SEND_FROM: Origin email address for the API.
 - CSM_EMAIL_APP_USERNAME: Username for the SMTP server.
 - CSM_EMAIL_APP_PASSWORD: Password for the SMTP server.

docker-compose-addons.yml

@@ -1,150 +1,17 @@
 version: '3'
-services:
-  cvmanager_api:
-    build:
-      context: ./services
-      dockerfile: Dockerfile.api
-    image: jpo_cvmanager_api:latest
-    restart: always
-    networks:
-      internal:
-    extra_hosts:
-      ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP}
-      ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP}
-    ports:
-      - '8081:5000'
-    environment:
-      PG_DB_HOST: ${PG_DB_HOST}
-      PG_DB_USER: ${PG_DB_USER}
-      PG_DB_PASS: ${PG_DB_PASS}
-      PG_DB_NAME: postgres
-      INSTANCE_CONNECTION_NAME: ${INSTANCE_CONNECTION_NAME}
-
-      MONGO_DB_URI: ${MONGO_DB_URI}
-      MONGO_DB_NAME: ${MONGO_DB_NAME}
-
-      COUNTS_MSG_TYPES: ${COUNTS_MSG_TYPES}
-      COUNTS_DB_TYPE: ${COUNTS_DB_TYPE}
-      COUNTS_DB_NAME: ${COUNTS_DB_NAME}
-      GOOGLE_APPLICATION_CREDENTIALS: '/google/gcp_service_account.json'
-
-      BSM_DB_NAME: ${BSM_DB_NAME}
-      SSM_DB_NAME: ${SSM_DB_NAME}
-      SRM_DB_NAME: ${SRM_DB_NAME}
-
-      WZDX_API_KEY: ${WZDX_API_KEY}
-      WZDX_ENDPOINT: ${WZDX_ENDPOINT}
-
-      CORS_DOMAIN: ${CORS_DOMAIN}
-      KEYCLOAK_ENDPOINT: http://${KEYCLOAK_DOMAIN}:8084/
-      KEYCLOAK_REALM: ${KEYCLOAK_REALM}
-      KEYCLOAK_API_CLIENT_ID: ${KEYCLOAK_API_CLIENT_ID}
-      KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY}
-
-      CSM_EMAIL_TO_SEND_FROM: ${CSM_EMAIL_TO_SEND_FROM}
-      CSM_EMAIL_APP_USERNAME: ${CSM_EMAIL_APP_USERNAME}
-      CSM_EMAIL_APP_PASSWORD: ${CSM_EMAIL_APP_PASSWORD}
-      CSM_EMAILS_TO_SEND_TO: ${CSM_EMAILS_TO_SEND_TO}
-      CSM_TARGET_SMTP_SERVER_ADDRESS: ${CSM_TARGET_SMTP_SERVER_ADDRESS}
-      CSM_TARGET_SMTP_SERVER_PORT: ${CSM_TARGET_SMTP_SERVER_PORT}
-
-      TIMEZONE: ${TIMEZONE}
-      LOGGING_LEVEL: ${API_LOGGING_LEVEL}
-    volumes:
-      - ./resources/google:/google
-    logging:
-      options:
-        max-size: '10m'
-        max-file: '5'
-
-  cvmanager_webapp:
-    build:
-      context: webapp
-      dockerfile: Dockerfile
-      args:
-        API_URI: http://${WEBAPP_DOMAIN}:8081
-        MAPBOX_TOKEN: ${MAPBOX_TOKEN}
-        KEYCLOAK_HOST_URL: http://${KEYCLOAK_DOMAIN}:8084/
-        COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES}
-        DOT_NAME: ${DOT_NAME}
-        MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE}
-        MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE}
-        MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM}
-    image: jpo_cvmanager_webapp:latest
-    restart: always
-    depends_on:
-      cvmanager_keycloak:
-        condition: service_healthy
-    networks:
-      internal:
-    extra_hosts:
-      ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP}
-      ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP}
-    ports:
-      - '80:80'
-    logging:
-      options:
-        max-size: '10m'
-
-  cvmanager_postgres:
-    image: postgis/postgis:15-master
-    restart: always
-    ports:
-      - '5432:5432'
-    environment:
-      POSTGRES_USER: ${PG_DB_USER}
-      POSTGRES_PASSWORD: ${PG_DB_PASS}
-    volumes:
-      - pgdb:/var/lib/postgresql/data
-      - ./resources/sql_scripts:/docker-entrypoint-initdb.d
-    logging:
-      options:
-        max-size: '10m'
 
-  cvmanager_keycloak:
-    build: ./resources/keycloak
-    image: jpo_cvmanager_keycloak:21.1
-    restart: always
-    depends_on:
-      - cvmanager_postgres
-    networks:
-      internal:
-    extra_hosts:
-      ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP}
-      ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP}
-    ports:
-      - '8084:8080'
-    environment:
-      KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
-      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
-      WEBAPP_ORIGIN: http://${WEBAPP_DOMAIN}
-      KC_HEALTH_ENABLED: true
-      KC_DB: postgres
-      KC_DB_URL: jdbc:postgresql://${PG_DB_HOST}/postgres?currentSchema=keycloak
-      KC_DB_USERNAME: ${PG_DB_USER}
-      KC_DB_PASSWORD: ${PG_DB_PASS}
-      KC_HOSTNAME: ${KEYCLOAK_DOMAIN}
-      KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY}
-      GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID}
-      GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET}
-    command:
-      - start-dev
-      - --log-level=${KC_LOGGING_LEVEL}
-      - --import-realm
-    volumes:
-      - ./resources/keycloak/realm.json:/opt/keycloak/data/import/realm.json
-      - ./resources/keycloak/${KEYCLOAK_LOGIN_THEME_NAME}.jar:/opt/keycloak/providers/theme.jar
-    logging:
-      options:
-        max-size: '10m'
+include:
+  - docker-compose.yml
 
+services:
   # ADDONS:
   jpo_bsm_query:
     build:
       context: ./services
       dockerfile: Dockerfile.bsm_query
     image: bsm_query:latest
     restart: always
+
     env_file:
       - ./services/addons/images/bsm_query/.env
     logging:
@@ -158,6 +25,7 @@ services:
       dockerfile: Dockerfile.count_metric
     image: count_metric:latest
     restart: always
+
     env_file:
       - ./services/addons/images/count_metric/.env
     logging:
@@ -171,10 +39,11 @@ services:
       dockerfile: Dockerfile.rsu_ping_fetch
     image: rsu_ping_fetch:latest
     restart: always
+
     depends_on:
       - cvmanager_postgres
     env_file:
-      - ./services/addons/images/rsu_ping_fetch/.env
+      - ./services/addons/images/rsu_ping/.env
     logging:
       options:
         max-size: '10m'
@@ -186,6 +55,7 @@ services:
       dockerfile: Dockerfile.iss_health_check
     image: iss_health_check:latest
     restart: always
+
     depends_on:
       - cvmanager_postgres
     env_file:
@@ -195,9 +65,31 @@ services:
         max-size: '10m'
         max-file: '5'
 
-volumes:
-  pgdb:
-    driver: local
+  firmware_manager:
+    build:
+      context: services
+      dockerfile: Dockerfile.firmware_manager
+    image: jpo_firmware_manager:latest
+    restart: always
+
+    ports:
+      - '8089:8080'
+    environment:
+      PG_DB_HOST: ${PG_DB_HOST}
+      PG_DB_NAME: postgres
+      PG_DB_USER: ${PG_DB_USER}
+      PG_DB_PASS: ${PG_DB_PASS}
 
-networks:
-  internal:
+      BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER}
+      BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET}
+
+      GCP_PROJECT: ${GCP_PROJECT}
+      GOOGLE_APPLICATION_CREDENTIALS: '/google/gcp_credentials.json'
+
+      LOGGING_LEVEL: ${API_LOGGING_LEVEL}
+    volumes:
+      - ${GOOGLE_APPLICATION_CREDENTIALS}:/google/gcp_credentials.json
+    logging:
+      options:
+        max-size: '10m'
+        max-file: '5'

docker-compose-webapp-deployment.yml

@@ -0,0 +1,23 @@
+# This file is used to build the webapp image for deployment.
+# The COUNTS_MSG_TYPES and DOT_NAME variables must be set in .env before building to populate
+# correctly in the deployed webapp as they are build-time variables.
+version: '3'
+services:
+  cvmanager_webapp:
+    build:
+      context: webapp
+      dockerfile: Dockerfile
+      args:
+        API_URI: ${WEBAPP_DOMAIN} # e.g. http://localhost
+        MAPBOX_TOKEN: ${MAPBOX_TOKEN}
+        KEYCLOAK_HOST_URL: ${KEYCLOAK_DOMAIN} # e.g. http://localhost
+        COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES}
+        DOT_NAME: ${DOT_NAME}
+        MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE}
+        MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE}
+        MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM}
+    image: jpo_cvmanager_webapp:latest
+    restart: always
+    logging:
+      options:
+        max-size: '10m'