Reasonable defaults, isolate tests, EOS_LOGGING_LEVEL, EOS_CONFIG_DIR

 * Add EOS_CONFIG_DIR to set config dir (relative path to EOS_DIR or
   absolute path).
    - config_folder_path read-only
    - config_file_path read-only
 * Default values to support app start with empty config:
    - latitude/longitude (Berlin)
    - optimization_ev_available_charge_rates_percent (null, so model
      default value is used)
    - Enable Akkudoktor electricity price forecast (docker-compose).
 * Fix some endpoints (empty data, remove unused params, fix types).
 * cacheutil: Use cache dir. Closes #240
 * Support EOS_LOGGING_LEVEL environment variable to set log level.
 * tests: All tests use separate temporary config
    - Add pytest switch --check-config-side-effect to check user
      config file existence after each test. Will also fail if user config
      existed before test execution (but will only check after the test has
      run).
      Enable flag in github workflow.
    - Globally mock platformdirs in config module. Now no longer required
      to patch individually.
      Function calls to config instance (e.g. merge_settings_from_dict)
      were unaffected previously.
 * Set Berlin as default location (default config/docker-compose).

Author: Lasall

.github/workflows/pytest.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Run Pytest
         run: |
           pip install -e .
-          python -m pytest --full-run -vs --cov src --cov-report term-missing
+          python -m pytest --full-run --check-config-side-effect -vs --cov src --cov-report term-missing
 
       - name: Upload test artifacts
         uses: actions/upload-artifact@v4

Dockerfile

@@ -9,6 +9,7 @@ ENV MPLCONFIGDIR="/tmp/mplconfigdir"
 ENV EOS_DIR="/opt/eos"
 ENV EOS_CACHE_DIR="${EOS_DIR}/cache"
 ENV EOS_OUTPUT_DIR="${EOS_DIR}/output"
+ENV EOS_CONFIG_DIR="${EOS_DIR}/config"
 
 WORKDIR ${EOS_DIR}
 
@@ -18,7 +19,9 @@ RUN adduser --system --group --no-create-home eos \
     && mkdir -p "${EOS_CACHE_DIR}" \
     && chown eos "${EOS_CACHE_DIR}" \
     && mkdir -p "${EOS_OUTPUT_DIR}" \
-    && chown eos "${EOS_OUTPUT_DIR}"
+    && chown eos "${EOS_OUTPUT_DIR}" \
+    && mkdir -p "${EOS_CONFIG_DIR}" \
+    && chown eos "${EOS_CONFIG_DIR}"
 
 COPY requirements.txt .
 
@@ -34,4 +37,4 @@ EXPOSE 8503
 
 CMD ["python", "-m", "akkudoktoreos.server.fastapi_server"]
 
-VOLUME ["${MPLCONFIGDIR}", "${EOS_CACHE_DIR}", "${EOS_OUTPUT_DIR}"]
+VOLUME ["${MPLCONFIGDIR}", "${EOS_CACHE_DIR}", "${EOS_OUTPUT_DIR}", "${EOS_CONFIG_DIR}"]

docker-compose.yaml

@@ -11,7 +11,11 @@ services:
       dockerfile: "Dockerfile"
       args:
         PYTHON_VERSION: "${PYTHON_VERSION}"
-    volumes:
-      - ./src/akkudoktoreos/default.config.json:/opt/eos/EOS.config.json:ro
+    environment:
+      - EOS_CONFIG_DIR=config
+      - latitude=52.2
+      - longitude=13.4
+      - elecprice_provider=ElecPriceAkkudoktor
+      - elecprice_charges=0.21
     ports:
       - "${EOS_PORT}:${EOS_PORT}"

docs/akkudoktoreos/openapi.json

@@ -778,26 +778,8 @@
     },
     "/pvforecast": {
       "get": {
-        "summary": "Fastapi Pvprognose",
-        "operationId": "fastapi_pvprognose_pvforecast_get",
-        "parameters": [
-          {
-            "name": "ac_power_measurement",
-            "in": "query",
-            "required": false,
-            "schema": {
-              "anyOf": [
-                {
-                  "type": "number"
-                },
-                {
-                  "type": "null"
-                }
-              ],
-              "title": "Ac Power Measurement"
-            }
-          }
-        ],
+        "summary": "Fastapi Pvforecast",
+        "operationId": "fastapi_pvforecast_pvforecast_get",
         "responses": {
           "200": {
             "description": "Successful Response",
@@ -808,16 +790,6 @@
                 }
               }
             }
-          },
-          "422": {
-            "description": "Validation Error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/HTTPValidationError"
-                }
-              }
-            }
           }
         }
       }
@@ -2987,32 +2959,6 @@
             "description": "Sub-path for the EOS cache data directory.",
             "default": "cache"
           },
-          "config_folder_path": {
-            "anyOf": [
-              {
-                "type": "string",
-                "format": "path"
-              },
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Config Folder Path",
-            "description": "Path to EOS configuration directory."
-          },
-          "config_file_path": {
-            "anyOf": [
-              {
-                "type": "string",
-                "format": "path"
-              },
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Config File Path",
-            "description": "Path to EOS configuration file."
-          },
           "pvforecast_planes": {
             "items": {
               "type": "string"
@@ -3100,6 +3046,34 @@
             "description": "Compute data_cache_path based on data_folder_path.",
             "readOnly": true
           },
+          "config_folder_path": {
+            "anyOf": [
+              {
+                "type": "string",
+                "format": "path"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Config Folder Path",
+            "description": "Path to EOS configuration directory.",
+            "readOnly": true
+          },
+          "config_file_path": {
+            "anyOf": [
+              {
+                "type": "string",
+                "format": "path"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Config File Path",
+            "description": "Path to EOS configuration file.",
+            "readOnly": true
+          },
           "config_default_file_path": {
             "type": "string",
             "format": "path",
@@ -3128,11 +3102,13 @@
           "timezone",
           "data_output_path",
           "data_cache_path",
+          "config_folder_path",
+          "config_file_path",
           "config_default_file_path",
           "config_keys"
         ],
         "title": "ConfigEOS",
-        "description": "Singleton configuration handler for the EOS application.\n\nConfigEOS extends `SettingsEOS` with support for  default configuration paths and automatic\ninitialization.\n\n`ConfigEOS` ensures that only one instance of the class is created throughout the application,\nallowing consistent access to EOS configuration settings. This singleton instance loads\nconfiguration data from a predefined set of directories or creates a default configuration if\nnone is found.\n\nInitialization Process:\n  - Upon instantiation, the singleton instance attempts to load a configuration file in this order:\n    1. The directory specified by the `EOS_DIR` environment variable.\n    2. A platform specific default directory for EOS.\n    3. The current working directory.\n  - The first available configuration file found in these directories is loaded.\n  - If no configuration file is found, a default configuration file is created in the platform\n    specific default directory, and default settings are loaded into it.\n\nAttributes from the loaded configuration are accessible directly as instance attributes of\n`ConfigEOS`, providing a centralized, shared configuration object for EOS.\n\nSingleton Behavior:\n  - This class uses the `SingletonMixin` to ensure that all requests for `ConfigEOS` return\n    the same instance, which contains the most up-to-date configuration. Modifying the configuration\n    in one part of the application reflects across all references to this class.\n\nAttributes:\n    _settings (ClassVar[SettingsEOS]): Holds application-wide settings.\n    _file_settings (ClassVar[SettingsEOS]): Stores configuration loaded from file.\n    config_folder_path (Optional[Path]): Path to the configuration directory.\n    config_file_path (Optional[Path]): Path to the configuration file.\n\nRaises:\n    FileNotFoundError: If no configuration file is found, and creating a default configuration fails.\n\nExample:\n    To initialize and access configuration attributes (only one instance is created):\n    ```python\n    config_eos = ConfigEOS()  # Always returns the same instance\n    print(config_eos.prediction_hours)  # Access a setting from the loaded configuration\n    ```"
+        "description": "Singleton configuration handler for the EOS application.\n\nConfigEOS extends `SettingsEOS` with support for  default configuration paths and automatic\ninitialization.\n\n`ConfigEOS` ensures that only one instance of the class is created throughout the application,\nallowing consistent access to EOS configuration settings. This singleton instance loads\nconfiguration data from a predefined set of directories or creates a default configuration if\nnone is found.\n\nInitialization Process:\n  - Upon instantiation, the singleton instance attempts to load a configuration file in this order:\n    1. The directory specified by the `EOS_CONFIG_DIR` environment variable\n    2. The directory specified by the `EOS_DIR` environment variable.\n    3. A platform specific default directory for EOS.\n    4. The current working directory.\n  - The first available configuration file found in these directories is loaded.\n  - If no configuration file is found, a default configuration file is created in the platform\n    specific default directory, and default settings are loaded into it.\n\nAttributes from the loaded configuration are accessible directly as instance attributes of\n`ConfigEOS`, providing a centralized, shared configuration object for EOS.\n\nSingleton Behavior:\n  - This class uses the `SingletonMixin` to ensure that all requests for `ConfigEOS` return\n    the same instance, which contains the most up-to-date configuration. Modifying the configuration\n    in one part of the application reflects across all references to this class.\n\nAttributes:\n    _settings (ClassVar[SettingsEOS]): Holds application-wide settings.\n    _file_settings (ClassVar[SettingsEOS]): Stores configuration loaded from file.\n    config_folder_path (Optional[Path]): Path to the configuration directory.\n    config_file_path (Optional[Path]): Path to the configuration file.\n\nRaises:\n    FileNotFoundError: If no configuration file is found, and creating a default configuration fails.\n\nExample:\n    To initialize and access configuration attributes (only one instance is created):\n    ```python\n    config_eos = ConfigEOS()  # Always returns the same instance\n    print(config_eos.prediction_hours)  # Access a setting from the loaded configuration\n    ```"
       },
       "ElectricVehicleParameters": {
         "properties": {

src/akkudoktoreos/config/config.py

@@ -14,7 +14,7 @@
 from pathlib import Path
 from typing import Any, ClassVar, List, Optional
 
-import platformdirs
+from platformdirs import user_config_dir, user_data_dir
 from pydantic import Field, ValidationError, computed_field
 
 # settings
@@ -40,6 +40,24 @@
 logger = get_logger(__name__)
 
 
+def get_absolute_path(
+    basepath: Optional[Path | str], subpath: Optional[Path | str]
+) -> Optional[Path]:
+    """Get path based on base path."""
+    if isinstance(basepath, str):
+        basepath = Path(basepath)
+    if subpath is None:
+        return basepath
+
+    if isinstance(subpath, str):
+        subpath = Path(subpath)
+    if subpath.is_absolute():
+        return subpath
+    if basepath is not None:
+        return basepath.joinpath(subpath)
+    return None
+
+
 class ConfigCommonSettings(SettingsBaseModel):
     """Settings for common configuration."""
 
@@ -60,22 +78,14 @@ class ConfigCommonSettings(SettingsBaseModel):
     @property
     def data_output_path(self) -> Optional[Path]:
         """Compute data_output_path based on data_folder_path."""
-        if self.data_output_subpath is None:
-            return self.data_folder_path
-        if self.data_folder_path and self.data_output_subpath:
-            return self.data_folder_path.joinpath(self.data_output_subpath)
-        return None
+        return get_absolute_path(self.data_folder_path, self.data_output_subpath)
 
     # Computed fields
     @computed_field  # type: ignore[prop-decorator]
     @property
     def data_cache_path(self) -> Optional[Path]:
         """Compute data_cache_path based on data_folder_path."""
-        if self.data_cache_subpath is None:
-            return self.data_folder_path
-        if self.data_folder_path and self.data_cache_subpath:
-            return self.data_folder_path.joinpath(self.data_cache_subpath)
-        return None
+        return get_absolute_path(self.data_folder_path, self.data_cache_subpath)
 
 
 class SettingsEOS(
@@ -114,9 +124,10 @@ class ConfigEOS(SingletonMixin, SettingsEOS):
 
     Initialization Process:
       - Upon instantiation, the singleton instance attempts to load a configuration file in this order:
-        1. The directory specified by the `EOS_DIR` environment variable.
-        2. A platform specific default directory for EOS.
-        3. The current working directory.
+        1. The directory specified by the `EOS_CONFIG_DIR` environment variable
+        2. The directory specified by the `EOS_DIR` environment variable.
+        3. A platform specific default directory for EOS.
+        4. The current working directory.
       - The first available configuration file found in these directories is loaded.
       - If no configuration file is found, a default configuration file is created in the platform
         specific default directory, and default settings are loaded into it.
@@ -150,21 +161,29 @@ class ConfigEOS(SingletonMixin, SettingsEOS):
     APP_NAME: ClassVar[str] = "net.akkudoktor.eos"  # reverse order
     APP_AUTHOR: ClassVar[str] = "akkudoktor"
     EOS_DIR: ClassVar[str] = "EOS_DIR"
+    EOS_CONFIG_DIR: ClassVar[str] = "EOS_CONFIG_DIR"
     ENCODING: ClassVar[str] = "UTF-8"
     CONFIG_FILE_NAME: ClassVar[str] = "EOS.config.json"
 
     _settings: ClassVar[Optional[SettingsEOS]] = None
     _file_settings: ClassVar[Optional[SettingsEOS]] = None
 
-    config_folder_path: Optional[Path] = Field(
-        None, description="Path to EOS configuration directory."
-    )
-
-    config_file_path: Optional[Path] = Field(
-        default=None, description="Path to EOS configuration file."
-    )
+    _config_folder_path: Optional[Path] = None
+    _config_file_path: Optional[Path] = None
 
     # Computed fields
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def config_folder_path(self) -> Optional[Path]:
+        """Path to EOS configuration directory."""
+        return self._config_folder_path
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def config_file_path(self) -> Optional[Path]:
+        """Path to EOS configuration file."""
+        return self._config_file_path
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def config_default_file_path(self) -> Path:
@@ -297,7 +316,7 @@ def _update_data_folder_path(self) -> None:
                 pass
         # From platform specific default path
         try:
-            data_dir = platformdirs.user_data_dir(self.APP_NAME, self.APP_AUTHOR)
+            data_dir = Path(user_data_dir(self.APP_NAME, self.APP_AUTHOR))
             if data_dir is not None:
                 data_dir.mkdir(parents=True, exist_ok=True)
                 self.data_folder_path = data_dir
@@ -308,62 +327,42 @@ def _update_data_folder_path(self) -> None:
         data_dir = Path.cwd()
         self.data_folder_path = data_dir
 
-    def _config_folder_path(self) -> Optional[Path]:
-        """Finds the first directory containing a valid configuration file.
+    def _get_config_file_path(self) -> tuple[Path, bool]:
+        """Finds the a valid configuration file or returns the desired path for a new config file.
 
         Returns:
-            Path: The path to the configuration directory, or None if not found.
+            tuple[Path, bool]: The path to the configuration directory and if there is already a config file there
         """
         config_dirs = []
-        config_dir = None
-        env_dir = os.getenv(self.EOS_DIR)
-        logger.debug(f"Envionment '{self.EOS_DIR}': '{env_dir}'")
+        env_base_dir = os.getenv(self.EOS_DIR)
+        env_config_dir = os.getenv(self.EOS_CONFIG_DIR)
+        env_dir = get_absolute_path(env_base_dir, env_config_dir)
+        logger.debug(f"Envionment config dir: '{env_dir}'")
         if env_dir is not None:
-            config_dirs.append(Path(env_dir).resolve())
-        config_dirs.append(Path(platformdirs.user_config_dir(self.APP_NAME)))
+            config_dirs.append(env_dir.resolve())
+        config_dirs.append(Path(user_config_dir(self.APP_NAME)))
         config_dirs.append(Path.cwd())
         for cdir in config_dirs:
             cfile = cdir.joinpath(self.CONFIG_FILE_NAME)
             if cfile.exists():
                 logger.debug(f"Found config file: '{cfile}'")
-                config_dir = cdir
-                break
-        return config_dir
-
-    def _config_file_path(self) -> Path:
-        """Finds the path to the configuration file.
-
-        Returns:
-            Path: The path to the configuration file. May not exist.
-        """
-        config_file = None
-        config_dir = self._config_folder_path()
-        if config_dir is None:
-            # There is currently no configuration file - create it in default path
-            env_dir = os.getenv(self.EOS_DIR)
-            if env_dir is not None:
-                config_dir = Path(env_dir).resolve()
-            else:
-                config_dir = Path(platformdirs.user_config_dir(self.APP_NAME))
-            config_file = config_dir.joinpath(self.CONFIG_FILE_NAME)
-        else:
-            config_file = config_dir.joinpath(self.CONFIG_FILE_NAME)
-        return config_file
+                return cfile, True
+        return config_dirs[0].joinpath(self.CONFIG_FILE_NAME), False
 
     def from_config_file(self) -> None:
         """Loads the configuration file settings for EOS.
 
         Raises:
             ValueError: If the configuration file is invalid or incomplete.
         """
-        config_file = self._config_file_path()
+        config_file, exists = self._get_config_file_path()
         config_dir = config_file.parent
-        if not config_file.exists():
+        if not exists:
             config_dir.mkdir(parents=True, exist_ok=True)
             try:
                 shutil.copy2(self.config_default_file_path, config_file)
             except Exception as exc:
-                logger.warning(f"Could not copy default config: {exc}. Using default copy...")
+                logger.warning(f"Could not copy default config: {exc}. Using default config...")
                 config_file = self.config_default_file_path
                 config_dir = config_file.parent
 
@@ -376,8 +375,8 @@ def from_config_file(self) -> None:
 
         self.update()
         # Everthing worked, remember the values
-        self.config_folder_path = config_dir
-        self.config_file_path = config_file
+        self._config_folder_path = config_dir
+        self._config_file_path = config_file
 
     def to_config_file(self) -> None:
         """Saves the current configuration to the configuration file.