Content analysis script improvements. (#13)

# Changes
- Print a message summary at the end of `raw_analysis.py`
- Consistency cleanup in `p1bin_types.py`, and added additional types

# Fixes
- Fixed Python import path in `p1bin_analysis.py`

bin/p1bin_analysis.py

@@ -6,6 +6,10 @@
 from fusion_engine_client.utils.argument_parser import ArgumentParser
 from fusion_engine_client.utils.log import DEFAULT_LOG_BASE_DIR, find_log_file
 
+# Add the parent directory to the search path to enable p1_runner package imports when not installed in Python.
+repo_root = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(repo_root)
+
 from p1_runner import trace as logging
 from p1_runner.p1bin_reader import P1BinReader, P1BinType
 from p1_runner.p1bin_type import find_matching_p1bin_types
@@ -26,10 +30,10 @@ def main():
         "file does not exist, do not generate one. Otherwise, a .p1i file will be created automatically to "
         "improve data read speed in the future.")
     parser.add_argument('--log-base-dir', metavar='DIR', default=DEFAULT_LOG_BASE_DIR,
-                        help="The base directory containing FusionEngine logs to be searched if a log pattern is"
+                        help="The base directory containing FusionEngine logs to be searched if a log pattern is "
                              "specified.")
     parser.add_argument('-o', '--output', type=str, metavar='DIR',
-                        help="The directory where output will be stored. Defaults to the parent directory of the input"
+                        help="The directory where output will be stored. Defaults to the parent directory of the input "
                              "file, or to the log directory if reading from a log.")
     parser.add_argument('-p', '--prefix', type=str,
                         help="Use the specified prefix for the output file: `<prefix>.p1log`. Otherwise, use the "

bin/raw_analysis.py

@@ -103,7 +103,7 @@ def index_messages(input_path, options):
                     else:
                         _logger.info(
                             f'Using existing index "{index_file_to_load}".')
-                        return index
+                        return index, file_size
 
     _logger.info(f"Indexing raw input.")
 
@@ -153,7 +153,13 @@ def index_messages(input_path, options):
 
         timestamp_fd.write(f'{EOF_FORMAT},0,{bytes_to_process},0,\n')
 
-    return load_index(index_file)
+    total_bytes_read = bytes_to_process
+    _logger.log(logging.INFO,
+                'Processed %d/%d bytes (%.1f%%). [elapsed=%.1f sec, rate=%.1f MB/s]' %
+                (total_bytes_read, bytes_to_process, 100.0 * float(total_bytes_read) / bytes_to_process,
+                    elapsed_sec, total_bytes_read / elapsed_sec / 1e6))
+
+    return load_index(index_file), file_size
 
 
 def load_index(index_file):
@@ -266,7 +272,8 @@ def generate_separated_logs(input_path, indexes, options):
                     help="If set, separate the RTCM contents into separate files each time the base station changes. "
                          "The file names will end with '_N.rtcm' where N is the the count of base stations seen.")
 parser.add_argument('--check-gaps', action=ExtendedBooleanAction, default=True,
-                    help="If set, separate the contents of each format type into their own files.")
+                    help="If set, search for unframed bytes that do not belong to a complete message from any "
+                         "protocol, indicating the existence of a gap in the data stream.")
 parser.add_argument('log',
                     help="The log to be read. May be one of:\n"
                     "- The path to a binary log file\n"
@@ -338,14 +345,32 @@ def raw_analysis(options):
         options.p1bin_type = [P1BinType.EXTERNAL_UNFRAMED_GNSS]
 
     logger.info(f"Output index stored in '{output_dir}'.")
-    index = index_messages(input_path, options)
+    index, file_size_bytes = index_messages(input_path, options)
 
     if options.check_gaps:
         find_gaps(index)
 
     if options.extract:
         generate_separated_logs(input_path, index, options)
 
+    _logger.info("")
+    format_string = '| {:<10} | {:>10} | {:>10} |'
+    _logger.info(format_string.format('Protocol', 'Messages', 'Bytes'))
+    _logger.info(format_string.format('-' * 10, '-' * 10, '-' * 10))
+    bytes_used = 0
+    for format in sorted(options.format):
+        message_length_bytes = [e[3] for e in index if e[0] == format]
+        format_bytes = sum(message_length_bytes)
+        bytes_used += format_bytes
+        _logger.info(format_string.format(format, len(message_length_bytes), format_bytes))
+
+    _logger.info("")
+    _logger.info(f"File size: {file_size_bytes} B")
+    processed_bytes = file_size_bytes - options.skip_bytes
+    if options.skip_bytes > 0:
+        _logger.info(f"File considered: {processed_bytes} B")
+    _logger.info(f"Unused: {processed_bytes - bytes_used} B")
+
 
 def extract_format(format):
     parser.description = f'Extract {format} contents of an input.raw or input.p1bin file.'

p1_runner/p1bin_type.py

@@ -10,11 +10,14 @@
 
 class P1BinType(IntEnum):
     INVALID = 0xFFFF
-    DEBUG = 0x01,
+    DEBUG = 0x01
     RTCM3_POLARIS = 0x21
-    M_TYPE_SBF = 0x40
+    SBF = 0x40
     EXTERNAL_UNFRAMED_GNSS = 0x42
-    M_TYPE_ATLAS_SBF = 0xa1
+    RTCM3_POLARIS_EPHEM = 0x77
+    ATLAS_SBF = 0xa1
+    FUSION_ENGINE_MESSAGE = 0xc6
+    RTCM3_UNKNOWN = 0x100
 
 
 class P1BinRecord(NamedTuple):