Adding support for CSV reports

Some users want to process report data, but generated PDF is not machine-readable.
CSV format can be easily imported into spreadsheets.
Adding optional configuration parameter for CSV generation.
Adding REPORT_NAME_NO_EXT variable for email template.

reports_module/etc/settings.yaml

@@ -32,6 +32,9 @@ reports:
   # Generated report pdf-files will be stored in this path
   report-path: /var/lib/xroad-metrics/reports
 
+  # Additionally generate csv reports
+  # generate-csv: False
+
   # Language translation config files are stored in this path
   translation-path: /etc/xroad-metrics/reports/lang
 
@@ -74,6 +77,14 @@ reports:
 
       Report name: {REPORT_NAME}
 
+      Produced services (CSV): csv/{REPORT_NAME_NO_EXT}_prod_serv.csv
+
+      Produced metaservices (CSV): csv/{REPORT_NAME_NO_EXT}_prod_meta.csv
+
+      Consumed services (CSV): csv/{REPORT_NAME_NO_EXT}_cons_serv.csv
+
+      Consumed metaservices (CSV): csv/{REPORT_NAME_NO_EXT}_cons_meta.csv
+
       X-Road subsystem: {X_ROAD_INSTANCE}:{MEMBER_CLASS}:{MEMBER_CODE}:{SUBSYSTEM_CODE}
 
       Report time period: from {START_DATE} to {END_DATE}

reports_module/opmon_reports/notification_manager.py

@@ -20,6 +20,7 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+import os
 import smtplib
 import ssl
 from typing import Iterable
@@ -85,7 +86,8 @@ def send_mail(self, notification: dict, receiver: dict):
             'MEMBER_CODE': notification['member_code'],
             'SUBSYSTEM_CODE': notification['subsystem_code'],
             'START_DATE': notification['start_date'],
-            'END_DATE': notification['end_date']
+            'END_DATE': notification['end_date'],
+            'REPORT_NAME_NO_EXT': os.path.splitext(notification['report_name'])[0]
         }
 
         msg = MIMEText(self.settings['message'].format(

reports_module/opmon_reports/report_manager.py

@@ -512,6 +512,35 @@ def save_pdf_to_file(self, html: str, creation_time):
 
         return report_name
 
+    def save_csv_to_file(self, data_frames, creation_time):
+        settings = self.reports_arguments.settings['reports']
+        output_directory = os.path.join(
+            settings['report-path'],
+            self.target.xroad_instance,
+            self.target.member_class,
+            self.target.member_code,
+            'csv'
+        )
+        if not os.path.isdir(output_directory):
+            os.makedirs(output_directory)
+
+        csv_suffixes = (
+            'prod_serv',
+            'prod_meta',
+            'cons_serv',
+            'cons_meta'
+        )
+
+        for i in range(len(csv_suffixes)):
+            report_name = (
+                f'{tools.format_string(self.target.subsystem_code)}_{self.start_date}_'
+                f'{self.end_date}_{creation_time}_{csv_suffixes[i]}.csv')
+            report_file = os.path.join(output_directory, report_name)
+
+            self.logger_manager.log_info(
+                'save_csv_to_file', f'Saving CSV report file "{report_file}".')
+            data_frames[i].to_csv(path_or_buf=report_file)
+
     def generate_report(self):
         start_generate_report = time.time()
 
@@ -524,6 +553,8 @@ def generate_report(self):
             creation_time = time_date_tools.datetime_to_modified_string(datetime.now())
             template = self.prepare_template(plots, data_frames, creation_time)
             report_name = self.save_pdf_to_file(template, creation_time)
+            if self.reports_arguments.settings['reports'].get('generate-csv', False):
+                self.save_csv_to_file(data_frames, creation_time)
 
         end_generate_report = time.time()
         total_time = time.strftime("%H:%M:%S", time.gmtime(end_generate_report - start_generate_report))