Updated utils.py and process_data.py to incorporate support for .MFC, .MFE, .MFM files (data compression and/or data encryption)

Author: Martin

.gitignore

@@ -9,4 +9,6 @@ output/
 *j1939-engine.dbc
 *FE34E37D*
 LOG2/
-*LIN.dbc
+*LIN.dbc
+*NMEA-2000-CSS-Electronics-v*
+*find_data_event.py

examples/data-processing/LOG/25674460/00000001/single_can_bus_1.MFC

@@ -28,6 +28,15 @@ fs = setup_fs(s3=True, key="access_key", secret="secret_key", endpoint="endpoint
 
 If you're using AWS S3, your endpoint would e.g. be `https://s3.us-east-2.amazonaws.com` (if your region is `us-east-2`). A MinIO S3 endpoint would e.g. be `http://192.168.0.1:9000`.
 
+---
+### Regarding encrypted log files
+If you need to handle encrypted log files, you can provide a passwords dictionary object with similar structure as the `passwords.json` file used in the CANedge MF4 converters. The object can be provided e.g. as below (or via environmental variables):
+
+```
+pw = {"default": "password"} 			# hardcoded  
+pw = json.load(open("passwords.json"))	# from local JSON file 
+```
+
 ---
 
 ### Regarding Transport Protocol example

examples/data-processing/LOG/25674460/00000001/single_can_bus_1.MFE

@@ -5,15 +5,17 @@
 from datetime import datetime, timezone
 from utils import setup_fs, load_dbc_files, restructure_data, add_custom_sig, ProcessData
 
-# specify devices to process (from local/S3), DBC files and start time
+# specify devices to process (from local/S3), DBC files, start time and optionally passwords
 devices = ["LOG/958D2219"]
+
 dbc_paths = ["dbc_files/CSS-Electronics-SAE-J1939-DEMO.dbc"]
 start = datetime(year=2020, month=1, day=13, hour=0, tzinfo=timezone.utc)
+pw = {"default": "password"}
 
 # setup filesystem (local/S3), load DBC files and list log files for processing
-fs = setup_fs(s3=False, key="", secret="", endpoint="")
+fs = setup_fs(s3=False, key="", secret="", endpoint="", passwords=pw)
 db_list = load_dbc_files(dbc_paths)
-log_files = canedge_browser.get_log_files(fs, devices, start_date=start)
+log_files = canedge_browser.get_log_files(fs, devices, start_date=start, passwords=pw)
 print(f"Found a total of {len(log_files)} log files")
 
 # --------------------------------------------
@@ -22,7 +24,7 @@
 df_phys_all = pd.DataFrame()
 
 for log_file in log_files:
-    df_raw, device_id = proc.get_raw_data(log_file)
+    df_raw, device_id = proc.get_raw_data(log_file, passwords=pw)
     df_phys = proc.extract_phys(df_raw)
     proc.print_log_summary(device_id, log_file, df_phys)
 

examples/data-processing/LOG/25674460/00000001/single_message_can_bus_1_1024_window.MFC

@@ -6,7 +6,7 @@ attrs==21.2.0
 bitstruct==8.11.1
 botocore==1.20.49
 can-decoder>=0.1.3
-canedge-browser>=0.0.7
+canedge-browser>=0.0.8
 canmatrix==0.9.1
 chardet==4.0.0
 click==8.0.1
@@ -17,7 +17,7 @@ idna==3.2
 importlib-metadata==4.5.0
 J1939-PGN==0.4
 jmespath==0.10.0
-mdf-iter>=0.0.4
+mdf-iter>=0.0.6
 multidict==5.1.0
 numpy==1.20.3
 pandas==1.2.4

examples/data-processing/LOG/25674460/00000001/single_message_can_bus_1_1024_window.MFM

@@ -1,4 +1,4 @@
-def setup_fs(s3, key="", secret="", endpoint="", cert=""):
+def setup_fs(s3, key="", secret="", endpoint="", cert="", passwords={}):
     """Given a boolean specifying whether to use local disk or S3, setup filesystem
     Syntax examples: AWS (http://s3.us-east-2.amazonaws.com), MinIO (http://192.168.0.1:9000)
     The cert input is relevant if you're using MinIO with TLS enabled, for specifying the path to the certficiate.
@@ -27,7 +27,7 @@ def setup_fs(s3, key="", secret="", endpoint="", cert=""):
         import canedge_browser
 
         base_path = Path(__file__).parent
-        fs = canedge_browser.LocalFileSystem(base_path=base_path)
+        fs = canedge_browser.LocalFileSystem(base_path=base_path, passwords=passwords)
 
     return fs
 
@@ -48,7 +48,7 @@ def load_dbc_files(dbc_paths):
 
 
 # -----------------------------------------------
-def list_log_files(fs, devices, start_times, verbose=True):
+def list_log_files(fs, devices, start_times, verbose=True, passwords={}):
     """Given a list of device paths, list log files from specified filesystem.
     Data is loaded based on the list of start datetimes
     """
@@ -59,7 +59,7 @@ def list_log_files(fs, devices, start_times, verbose=True):
     if len(start_times):
         for idx, device in enumerate(devices):
             start = start_times[idx]
-            log_files_device = canedge_browser.get_log_files(fs, [device], start_date=start)
+            log_files_device = canedge_browser.get_log_files(fs, [device], start_date=start, passwords=passwords)
             log_files.extend(log_files_device)
 
     if verbose:
@@ -180,14 +180,14 @@ def filter_signals(self, df_phys):
 
         return df_phys
 
-    def get_raw_data(self, log_file, lin=False):
+    def get_raw_data(self, log_file, lin=False, passwords={}):
         """Extract a df of raw data and device ID from log file.
         Optionally include LIN bus data by setting lin=True
         """
         import mdf_iter
 
         with self.fs.open(log_file, "rb") as handle:
-            mdf_file = mdf_iter.MdfFile(handle)
+            mdf_file = mdf_iter.MdfFile(handle, passwords=passwords)
             device_id = self.get_device_id(mdf_file)
 
             if lin: