Merge pull request #1473 from OpenC3/py_variable_size

Add multiple variable bit length items to python

.rubocop.yml

@@ -42,3 +42,6 @@ Lint/OrAssignmentToConstant:
   Enabled: false
 Lint/UnusedMethodArgument:
   Enabled: false
+# This rule doesn't allow for value.to_s(16)
+Lint/RedundantStringCoercion:
+  Enabled: false

openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/procedures/collect.py

@@ -11,7 +11,7 @@
 if not isinstance(number, (int, float)):
     raise RuntimeError("Bad return")
 number = ask_string("Enter a number.")
-if not type(number) == str:
+if not isinstance(number, str):
     raise RuntimeError("Bad return")
 
 result = message_box("Click something.", "CHOICE1", "CHOICE2")

openc3-cosmos-init/plugins/packages/openc3-tool-common/src/components/widgets/ImageviewerWidget.vue

@@ -43,7 +43,7 @@ export default {
     },
   },
   created: function () {
-    // TODO: We're hard coding COVERTED because the existing 4th parameter is the image format
+    // TODO: We're hard coding CONVERTED because the existing 4th parameter is the image format
     // Future breaking change would be to put the type as 4th and format as fifth
     this.valueId = `${this.parameters[0]}__${this.parameters[1]}__${this.parameters[2]}__CONVERTED`
     this.screen.addItem(this.valueId)

openc3-cosmos-script-runner-api/scripts/run_script.py

@@ -130,7 +130,7 @@ def run_script_log(id, message, color="BLACK", message_log=True):
     for msg in p.listen():
         parsed_cmd = json.loads(msg["data"])
         if not parsed_cmd == "shutdown" or (
-            type(parsed_cmd) is dict and parsed_cmd["method"]
+            isinstance(parsed_cmd, dict) and parsed_cmd["method"]
         ):
             run_script_log(id, f"Script {path} received command: {msg['data']}")
         match parsed_cmd:
@@ -148,7 +148,7 @@ def run_script_log(id, message, color="BLACK", message_log=True):
             case "shutdown":
                 p.unsubscribe()
             case _:
-                if type(parsed_cmd) is dict and "method" in parsed_cmd:
+                if isinstance(parsed_cmd, dict) and "method" in parsed_cmd:
                     match parsed_cmd["method"]:
                         # This list matches the list in running_script.py:113
                         case (

openc3-cosmos-script-runner-api/scripts/running_script.py

@@ -527,8 +527,8 @@ def post_line_instrumentation(self, filename, line_number):
     def exception_instrumentation(self, filename, line_number):
         _, error, _ = sys.exc_info()
         if (
-            issubclass(error.__class__, StopScript)
-            or issubclass(error.__class__, SkipScript)
+            isinstance(error, StopScript)
+            or isinstance(error, SkipScript)
             or not self.use_instrumentation
         ):
             raise error
@@ -994,9 +994,7 @@ def run_thread_body(
                 )
 
         except Exception as error:
-            if issubclass(error.__class__, StopScript) or issubclass(
-                error.__class__, SkipScript
-            ):
+            if isinstance(error, StopScript) or isinstance(error, SkipScript):
                 self.handle_output_io()
                 self.scriptrunner_puts(
                     f"Script stopped: {os.path.basename(self.filename)}"

openc3/lib/openc3/accessors/binary_accessor.rb

@@ -156,8 +156,10 @@ def handle_write_variable_bit_size(item, value, buffer)
           current_bit_size = 14
         when 2
           current_bit_size = 30
-        else
+        when 3
           current_bit_size = 62
+        else
+          raise "Value #{item.variable_bit_size['length_item_name']} has unknown QUIC bit size encoding: #{length_item_value}"
         end
 
         if item.data_type == :UINT

openc3/lib/openc3/conversions/bit_reverse_conversion.rb

@@ -21,10 +21,10 @@
 module OpenC3
   class BitReverseConversion < Conversion
     def initialize(converted_type, converted_bit_size)
+      super()
       @converted_type = converted_type.to_s.upcase.intern
       @converted_bit_size = converted_bit_size.to_i
-      @converted_array_size = nil
-      if @converted_data_type == :FLOAT
+      if @converted_type == :FLOAT
         raise "Float Bit Reverse Not Yet Supported"
       end
     end
@@ -48,7 +48,7 @@ def call(value, _packet, _buffer)
 
     # @return [String] The conversion class
     def to_s
-      "#{self.class.to_s.split('::')[-1]}.new(#{@converted_type}, #{@converted_bit_size})"
+      "#{self.class.to_s.split('::')[-1]}.new(:#{@converted_type}, #{@converted_bit_size})"
     end
 
     # @param read_or_write [String] Either 'READ' or 'WRITE'

openc3/lib/openc3/packets/structure.rb

@@ -618,7 +618,10 @@ def recalculate_bit_offsets
         if item.original_bit_offset >= 0
           item.bit_offset = item.original_bit_offset + adjustment
           if item.data_type != :DERIVED and (item.variable_bit_size or item.original_bit_size <= 0 or (item.original_array_size and item.original_array_size <= 0))
-            adjustment += calculate_total_bit_size(item)
+            new_bit_size = calculate_total_bit_size(item)
+            if item.original_bit_size != new_bit_size
+              adjustment += (new_bit_size - item.original_bit_size)
+            end
           end
         end
       end

openc3/python/openc3/accessors/binary_accessor.py

@@ -102,6 +102,38 @@ def raise_buffer_error(cls, read_write, buffer, data_type, given_bit_offset, giv
     # Valid endianess
     ENDIANNESS = ["BIG_ENDIAN", "LITTLE_ENDIAN"]
 
+    def handle_read_variable_bit_size(self, item, _buffer):
+        length_value = self.packet.read(item.variable_bit_size["length_item_name"], "CONVERTED")
+        if item.array_size is not None:
+            item.array_size = (length_value * item.variable_bit_size["length_bits_per_count"]) + item.variable_bit_size[
+                "length_value_bit_offset"
+            ]
+        else:
+            if item.data_type == "INT" or item.data_type == "UINT":
+                # QUIC encoding is currently assumed for individual variable sized integers
+                # see https://datatracker.ietf.org/doc/html/rfc9000#name-variable-length-integer-enc
+                match length_value:
+                    case 0:
+                        item.bit_size = 6
+                    case 1:
+                        item.bit_size = 14
+                    case 2:
+                        item.bit_size = 30
+                    case _:
+                        item.bit_size = 62
+            else:
+                item.bit_size = (
+                    length_value * item.variable_bit_size["length_bits_per_count"]
+                ) + item.variable_bit_size["length_value_bit_offset"]
+
+    def read_item(self, item, buffer):
+        if item.data_type == "DERIVED":
+            return None
+        if item.variable_bit_size:
+            self.handle_read_variable_bit_size(item, buffer)
+        return BinaryAccessor.class_read_item(item, buffer)
+
+    # Note: do not use directly - use instance read_item
     @classmethod
     def class_read_item(cls, item, buffer):
         if item.data_type == "DERIVED":
@@ -118,6 +150,142 @@ def class_read_item(cls, item, buffer):
         else:
             return cls.read(item.bit_offset, item.bit_size, item.data_type, buffer, item.endianness)
 
+    def handle_write_variable_bit_size(self, item, value, buffer):
+        adjustment = 0
+        # Update length field to new size
+        if (item.data_type == "INT" or item.data_type == "UINT") and item.original_array_size is None:
+            adjustment = self._write_variable_int(item, value, buffer)
+        # Probably not possible to get this condition because we don't allow 0 sized floats
+        # but check for it just to cover all the possible data_types
+        elif item.data_type == "FLOAT":
+            raise AttributeError("Variable bit size not currently supported for FLOAT data type")
+        else:
+            # STRING, BLOCK, or array types
+            adjustment = self._write_variable_other(item, value, buffer)
+
+        # Recalculate bit offsets after this item
+        if adjustment != 0 and item.bit_offset >= 0:
+            for sitem in self.packet.sorted_items:
+                if sitem.data_type == "DERIVED" or sitem.bit_offset < item.bit_offset:
+                    # Skip items before this item and derived items and items with negative bit offsets
+                    continue
+                if sitem != item:
+                    sitem.bit_offset += adjustment
+
+    def _write_variable_int(self, item, value, buffer):
+        # QUIC encoding is currently assumed for individual variable sized integers
+        # see https://datatracker.ietf.org/doc/html/rfc9000#name-variable-length-integer-enc
+
+        # Calculate current bit size so we can preserve bytes after the item
+        length_item_value = self.packet.read(item.variable_bit_size["length_item_name"], "CONVERTED")
+        match length_item_value:
+            case 0:
+                current_bit_size = 6
+            case 1:
+                current_bit_size = 14
+            case 2:
+                current_bit_size = 30
+            case 3:
+                current_bit_size = 62
+            case _:
+                raise AttributeError(
+                    f"Value {item.variable_bit_size['length_item_name']} has unknown QUIC bit size encoding: {length_item_value}"
+                )
+
+        if item.data_type == "UINT":
+            if value <= 63:
+                # Length = 0, value up to 6-bits
+                new_bit_size = 6
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 0)
+            elif value <= 16383:
+                # Length = 1, value up to 14-bits
+                new_bit_size = 14
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 1)
+            elif value <= 1073741823:
+                # Length = 2, value up to 30-bits
+                new_bit_size = 30
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 2)
+            else:
+                # Length = 3, value up to 62-bits
+                new_bit_size = 62
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 3)
+        else:
+            if value <= 31 and value >= -32:
+                # Length = 0, value up to 6-bits
+                new_bit_size = 6
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 0)
+            elif value <= 8191 and value >= -8192:
+                # Length = 1, value up to 14-bits
+                new_bit_size = 14
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 1)
+            elif value <= 536870911 and value >= -536870912:
+                # Length = 2, value up to 30-bits
+                new_bit_size = 30
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 2)
+            else:
+                # Length = 3, value up to 62-bits
+                new_bit_size = 62
+                item.bit_size = new_bit_size
+                self.packet.write(item.variable_bit_size["length_item_name"], 3)
+
+        # Later items need their bit_offset adjusted by the change in this item
+        adjustment = new_bit_size - current_bit_size
+        adjust_bytes = int(adjustment / 8)
+        item_offset = int(item.bit_offset / 8)
+        if adjust_bytes > 0:
+            original_length = len(buffer)
+            # Add extra bytes because we're adjusting larger
+            buffer += BinaryAccessor.ZERO_STRING * adjust_bytes
+            # We added bytes to the end so now we have to shift the buffer over
+            # We copy from the original offset with the original length
+            # to the new shifted offset all the way to the end of the buffer
+            buffer[(item_offset + adjust_bytes) :] = buffer[item_offset:original_length]
+        elif adjust_bytes < 0:
+            # Remove extra bytes because we're adjusting smaller
+            del buffer[item_offset + 1 : item_offset + 1 - adjust_bytes]
+
+        return adjustment
+
+    def _write_variable_other(self, item, value, buffer):
+        # Calculate current bit size so we can preserve bytes after the item
+        length_item_value = self.packet.read(item.variable_bit_size["length_item_name"], "CONVERTED")
+        current_bit_size = (
+            length_item_value * item.variable_bit_size["length_bits_per_count"]
+        ) + item.variable_bit_size["length_value_bit_offset"]
+
+        # Calculate bits after this item
+        bits_with_item = item.bit_offset + current_bit_size
+        bits_after_item = (len(buffer) * 8) - bits_with_item
+        if item.original_array_size is not None:
+            item.array_size = -bits_after_item
+        else:
+            item.bit_size = -bits_after_item
+
+        new_bit_size = len(value) * 8
+        length_value = (new_bit_size - item.variable_bit_size["length_value_bit_offset"]) / item.variable_bit_size[
+            "length_bits_per_count"
+        ]
+        self.packet.write(item.variable_bit_size["length_item_name"], length_value)
+
+        # Later items need their bit_offset adjusted by the change in this item
+        # so return the adjustment value
+        return new_bit_size - current_bit_size
+
+    def write_item(self, item, value, buffer):
+        if item.data_type == "DERIVED":
+            return None
+        if item.variable_bit_size:
+            self.handle_write_variable_bit_size(item, value, buffer)
+        BinaryAccessor.class_write_item(item, value, buffer)
+
+    # Note: do not use directly - use instance write_item
     @classmethod
     def class_write_item(cls, item, value, buffer):
         if item.data_type == "DERIVED":
@@ -185,7 +353,7 @@ def read(cls, bit_offset, bit_size, data_type, buffer, endianness):
                     try:
                         buffer = buffer[lower_bound : (upper_bound + 1)]
                         try:
-                            return buffer[: buffer.index(b"\00")].decode(encoding="utf-8")
+                            return buffer[: buffer.index(BinaryAccessor.ZERO_STRING)].decode(encoding="utf-8")
                         except ValueError:
                             return buffer.decode(encoding="utf-8")
                     # If this 'STRING' contains binary buffer.decode will fail
@@ -332,7 +500,7 @@ def write(cls, value, bit_offset, bit_size, data_type, buffer, endianness, overf
             #######################################
 
             if cls.byte_aligned(bit_offset):
-                if data_type == "STRING" and type(value) == str:
+                if data_type == "STRING" and isinstance(value, str):
                     temp = value.encode(encoding="utf-8")
                 else:
                     temp = value
@@ -347,7 +515,7 @@ def write(cls, value, bit_offset, bit_size, data_type, buffer, endianness, overf
                         # String was completely empty
                         if end_bytes > 0:
                             # Preserve bytes at end of buffer
-                            buffer += b"\00" * len(value)
+                            buffer += BinaryAccessor.ZERO_STRING * len(value)
                             lower_index = lower_bound + len(value)
                             buffer[lower_index : lower_index + end_bytes] = buffer[
                                 lower_bound : lower_bound + end_bytes
@@ -362,20 +530,20 @@ def write(cls, value, bit_offset, bit_size, data_type, buffer, endianness, overf
                     elif (upper_bound > old_upper_bound) and (end_bytes > 0):
                         # Preserve bytes at end of buffer
                         diff = upper_bound - old_upper_bound
-                        buffer += b"\00" * diff
+                        buffer += BinaryAccessor.ZERO_STRING * diff
                         buffer[upper_bound + 1 : upper_bound + 1 + end_bytes] = buffer[
                             old_upper_bound + 1 : old_upper_bound + 1 + end_bytes
                         ]
 
                 else:  # given_bit_size > 0
                     byte_size = math.floor(bit_size / 8)
                     if len(value) < byte_size:
-                        if type(value) is str:
+                        if isinstance(value, str):
                             ba = bytearray()
                             ba.extend(value.encode(encoding="utf-8"))
                             value = ba
                         # Pad the requested size with zeros
-                        temp = value.ljust(byte_size, b"\00")
+                        temp = value.ljust(byte_size, BinaryAccessor.ZERO_STRING)
                     elif len(value) > byte_size:
                         if overflow == "TRUNCATE":
                             # Resize the value to fit the field
@@ -800,7 +968,7 @@ def write_array(
         given_array_size = array_size
 
         # Verify a list was given
-        if type(values) != list:
+        if not isinstance(values, list):
             raise AttributeError(f"values must be a list but is {values.__class__.__name__}")
 
         # Handle negative and zero bit sizes
@@ -882,7 +1050,7 @@ def write_array(
                 if byte_aligned:
                     for index in range(num_writes):
                         value = values[index]
-                        if type(value) == int:
+                        if isinstance(value, int):
                             value = value.to_bytes(1, byteorder="big")
                         cls.write(
                             value,

openc3/python/openc3/accessors/cbor_accessor.py

@@ -23,7 +23,7 @@ class CborAccessor(JsonAccessor):
     def class_read_item(cls, item, buffer):
         if item.data_type == "DERIVED":
             return None
-        if type(buffer) is bytearray:
+        if isinstance(buffer, bytearray):
             parsed = loads(buffer)
         else:
             parsed = buffer
@@ -33,14 +33,14 @@ def class_read_item(cls, item, buffer):
     def class_write_item(cls, item, value, buffer):
         if item.data_type == "DERIVED":
             return None
-        if type(buffer) is bytearray:
+        if isinstance(buffer, bytearray):
             decoded = loads(buffer)
         else:
             decoded = buffer
 
         super().class_write_item(item, value, decoded)
 
-        if type(buffer) is bytearray:
+        if isinstance(buffer, bytearray):
             # buffer[0:] syntax so we copy into the buffer
             buffer[0:] = dumps(decoded)
         return value