From 3a1103739ff57300613d21d0fe6d61e4f73b582a Mon Sep 17 00:00:00 2001 From: Mohamed Elbeltagy Date: Mon, 13 May 2024 11:21:03 +0300 Subject: [PATCH 1/4] feat: Add ability to disable certificate verification during connection to central server Refs: OPMONDEV-181 --- collector_module/etc/settings.yaml | 8 +++++++- .../opmon_collector/central_server_client.py | 16 ++++++++++++---- docs/collector_module.md | 12 ++++++++---- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/collector_module/etc/settings.yaml b/collector_module/etc/settings.yaml index 06fba5e7..e95ab55f 100644 --- a/collector_module/etc/settings.yaml +++ b/collector_module/etc/settings.yaml @@ -67,6 +67,12 @@ xroad: protocol: http:// host: timeout: 10 + # path to client's certificate + tls-client-certificate: + # path to client's private key + tls-client-key: + # path to server's certificate, or False to disable server certificate verification + tls-server-certificate: # Security server used to contact security-server: @@ -77,7 +83,7 @@ xroad: tls-client-certificate: # path to client's private key tls-client-key: - # path to server's certificate + # path to server's certificate, or False to disable server certificate verification tls-server-certificate: # X-Road service configuration used to fetch operational monitoring requests. diff --git a/collector_module/opmon_collector/central_server_client.py b/collector_module/opmon_collector/central_server_client.py index eb904bb5..892f7d09 100644 --- a/collector_module/opmon_collector/central_server_client.py +++ b/collector_module/opmon_collector/central_server_client.py @@ -28,8 +28,14 @@ class CentralServerClient: def __init__(self, xroad_settings, logger_m): - self.url = f"{xroad_settings['central-server']['protocol']}{xroad_settings['central-server']['host']}" - self.timeout = xroad_settings['central-server']['timeout'] + central_server_settings = xroad_settings['central-server'] + self.url = f"{central_server_settings['protocol']}{central_server_settings['host']}" + self.timeout = central_server_settings['timeout'] + self.server_cert = central_server_settings.get('tls-server-certificate') + self.client_cert = ( + central_server_settings.get('tls-client-certificate'), + central_server_settings.get('tls-client-key') + ) self.logger_m = logger_m def get_security_servers(self): @@ -43,13 +49,15 @@ def get_security_servers(self): def _get_shared_params(self): internal_conf_url = f'{self.url}/internalconf' - global_conf = requests.get(internal_conf_url, timeout=self.timeout) + global_conf = requests.get(internal_conf_url, timeout=self.timeout, cert=self.client_cert, + verify=self.server_cert) global_conf.raise_for_status() # NB! re.search global configuration regex might be changed # according version naming or other future naming conventions data = global_conf.content.decode('utf-8') s = re.search(r'Content-location: (/V\d+/\d+/shared-params.xml)', data) - shared_params = requests.get(f'{self.url}{s.group(1)}', timeout=self.timeout) + shared_params = requests.get(f'{self.url}{s.group(1)}', timeout=self.timeout, + cert=self.client_cert, verify=self.server_cert) shared_params.raise_for_status() return shared_params diff --git a/docs/collector_module.md b/docs/collector_module.md index bfe68042..b771a68c 100644 --- a/docs/collector_module.md +++ b/docs/collector_module.md @@ -128,10 +128,14 @@ security-server: tls-client-key: /path/to/client.key # path to client's private key tls-server-certificate: /path/to/server.crt # path to server's certificate ``` -Notes: - Client's certificate has to be sent to security server administrator. - Server certificate has to be sent by server's administrator and save in client's location. - `tls-server-certificate` can be set to `False` to disable server certificate verification. + +> [!IMPORTANT] +> - Client's certificate has to be sent to security server administrator. +> - Server certificate has to be sent by server's administrator and save in client's location. + +> [!Note] +> **For non-production environments only**: +> To disable server certificate verification, set `tls-server-certificate` to `False`. ### Manual usage From 41af0758fc51a82ddd6b3d089f61f4cdaddd4425 Mon Sep 17 00:00:00 2001 From: Mohamed Elbeltagy Date: Mon, 13 May 2024 11:32:11 +0300 Subject: [PATCH 2/4] docs: Update README and collector module's documentation Refs: OPMONDEV-181 --- README.md | 14 ++++++++------ docs/collector_module.md | 40 +++++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 05023803..1c00bca4 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,14 @@ Instructions on setting up the MongoDB can be found in the [Database Module](./d The modules should be set up in the following order: -1. [Collector](./docs/collector_module.md) (before others) -2. [Corrector](./docs/corrector_module.md) (after Collector, before others) -3. [Reports](./docs/reports_module.md) (optional, after previous) -4. [Opendata](./docs/opendata_module.md) (optional, after previous) -5. [Networking](./docs/networking_module.md) (optional, after Opendata) -6. [Opendata Collector module](./docs/opendata_collector_module.md) (optional, after Opendata) + - [Database module](./docs/database_module.md) + - [Collector module](./docs/collector_module.md) + - [Corrector module](./docs/corrector_module.md) + - [Reports module](./docs/reports_module.md) + - [Anonymizer module](./docs/anonymizer_module.md) + - [Opendata module](./docs/opendata_module.md) + - [Networking/Visualizer module](./docs/networking_module.md) + - [Opendata Collector module](./docs/opendata_collector_module.md) ## Programming language diff --git a/docs/collector_module.md b/docs/collector_module.md index b771a68c..54316a20 100644 --- a/docs/collector_module.md +++ b/docs/collector_module.md @@ -1,6 +1,6 @@ -| [![X-ROAD](img/xroad-metrics-100.png)](https://x-road.global/) | ![European Union / European Regional Development Fund / Investing in your future](img/eu_rdf_100_en.png "Documents that are tagged with EU/SF logos must keep the logos until 1.11.2022. If it has not stated otherwise in the documentation. If new documentation is created using EU/SF resources the logos must be tagged appropriately so that the deadline for logos could be found.") | -| :-------------------------------------------------- | -------------------------: | +| [![X-ROAD](img/xroad-metrics-100.png)](https://x-road.global/) | ![European Union / European Regional Development Fund / Investing in your future](img/eu_rdf_100_en.png "Documents that are tagged with EU/SF logos must keep the logos until 1.11.2022. If it has not stated otherwise in the documentation. If new documentation is created using EU/SF resources the logos must be tagged appropriately so that the deadline for logos could be found.") | +|:---------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| # X-Road Metrics - Collector Module @@ -68,13 +68,13 @@ sudo apt-get install xroad-metrics-collector ``` The installation package automatically installs following items: - * xroad-metrics-collector command to run the collector manually + * `xroad-metrics-collector` command to run the collector manually * Linux user named _xroad-metrics_ and group _xroad-metrics_ * settings file _/etc/xroad-metrics/collector/settings.yaml_ * cronjob in _/etc/cron.d/xroad-metrics-collector-cron_ to run collector automatically every three hours * log folders to _/var/log/xroad-metrics/collector/_ -Only _xroad-metrics_ user can access the settings files and run xroad-metrics-collector command. +Only _xroad-metrics_ user can access the settings files and run `xroad-metrics-collector` command. To use collector you need to fill in your X-Road and MongoDB configuration into the settings file. Refer to section [Collector Configuration](#collector-configuration) @@ -92,23 +92,29 @@ To use collector you need to fill in your X-Road and MongoDB configuration into ```bash sudo vi /etc/xroad-metrics/collector/settings.yaml ``` +> [!TIP] +> For a complete list of available settings, please refer to this [settings.yaml](../collector_module/etc/settings.yaml) template file. Settings that the user must fill in: * X-Road instance name -* Central- and Security Server hosts +* Central and Security Server hosts * X-Road client used to collect the monitoring data * username and password for the collector module MongoDB user -To run collector for multiple X-Road instances, a settings profile for each instance can be created. For example to have profiles DEV, TEST and PROD create three copies of `setting.yaml` -file named `settings_DEV.yaml`, `settings_TEST.yaml` and `settings_PROD.yaml`. -Then fill the profile specific settings to each file and use the --profile -flag when running xroad-metrics-collector. For example to run using the TEST profile: -``` -xroad-metrics-collector --profile TEST collect -``` - -`xroad-metrics-collector` command searches the settings file first in current working direcrtory, then in -_/etc/xroad-metrics/collector/_ +#### Configurations for multiple X-Road instances + +To run collector for multiple X-Road instances, a settings profile for each instance can be created. +1. To have profiles `DEV`, `TEST` and `PROD`, create three copies of `setting.yaml` +file named `settings_DEV.yaml`, `settings_TEST.yaml` and `settings_PROD.yaml` respectively. +2. Fill the profile specific settings to each file. +3. Use the `--profile` flag when running `xroad-metrics-collector`. + For example, to run using the `TEST` profile: + ```shell + xroad-metrics-collector --profile TEST collect + ``` +> [!IMPORTANT] +> `xroad-metrics-collector` command searches the settings file first in current working directory, then in +`/etc/xroad-metrics/collector/` ### Using client certificate (mTLS) to connect to security server @@ -213,7 +219,7 @@ logger: ``` -The log file is written to `log-path` and log file name contains the X-Road instance name. +The log file is written to `log-path` and log file name contains the X-Road instance name. The above example configuration would write logs to `/var/log/xroad-metrics/collector/logs/log_collector_EXAMPLE.json`. Every log line includes: @@ -234,7 +240,7 @@ In case of "activity": "collector_end", the "msg" includes values separated by c The **collector module** log handler is compatible with the logrotate utility. To configure log rotation for the example setup above, create the file: -``` +```bash sudo vi /etc/logrotate.d/xroad-metrics-collector ``` From 77c4738cfd861ed0c6653c035406c6900155e247 Mon Sep 17 00:00:00 2001 From: Mohamed Elbeltagy Date: Tue, 14 May 2024 20:06:07 +0300 Subject: [PATCH 3/4] docs: Update collector module's documentation Refs: OPMONDEV-181 --- docs/collector_module.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/collector_module.md b/docs/collector_module.md index 54316a20..4656ca19 100644 --- a/docs/collector_module.md +++ b/docs/collector_module.md @@ -135,13 +135,10 @@ security-server: tls-server-certificate: /path/to/server.crt # path to server's certificate ``` -> [!IMPORTANT] +> [!Note] > - Client's certificate has to be sent to security server administrator. > - Server certificate has to be sent by server's administrator and save in client's location. - -> [!Note] -> **For non-production environments only**: -> To disable server certificate verification, set `tls-server-certificate` to `False`. +> - `tls-server-certificate` can be set to `False` to disable server certificate verification. ### Manual usage @@ -219,7 +216,7 @@ logger: ``` -The log file is written to `log-path` and log file name contains the X-Road instance name. +The log file is written to `log-path` and log file name contains the X-Road instance name. The above example configuration would write logs to `/var/log/xroad-metrics/collector/logs/log_collector_EXAMPLE.json`. Every log line includes: From 3ec48413ae199560ad8ffab433ab47f3ec83ce61 Mon Sep 17 00:00:00 2001 From: Mohamed Elbeltagy Date: Tue, 14 May 2024 20:33:54 +0300 Subject: [PATCH 4/4] fix: Stop using dictionary unpacking for single members to fix failing tests Refs: OPMONDEV-181 --- anonymizer_module/metrics_statistics/statistics_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/anonymizer_module/metrics_statistics/statistics_manager.py b/anonymizer_module/metrics_statistics/statistics_manager.py index 678ab061..f5132bd1 100644 --- a/anonymizer_module/metrics_statistics/statistics_manager.py +++ b/anonymizer_module/metrics_statistics/statistics_manager.py @@ -37,9 +37,9 @@ def collect_statistics(settings: dict, logger: Logger, output_only: bool = False statistics: StatisticalData = { **requests_counts, - **{'member_count': json.dumps(member_counts)}, - **{'service_count': len(services)}, - **{'service_request_count': json.dumps(services_counts)} + 'member_count': json.dumps(member_counts), + 'service_count': len(services), + 'service_request_count': json.dumps(services_counts) } if output_only: logger.info('Metrics statistical data:\n\n%s', pformat(statistics, indent=2, width=2))