Remove payload length cap from Eventstream messages

.changes/next-release/enhancement-Eventsteam-78927.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "Eventsteam",
+  "description": "The SDK no longer validates payload size for eventstreams. This is to facilitate varying payload requirements across AWS services."
+}

botocore/eventstream.py

@@ -20,7 +20,6 @@
 # byte length of the prelude (total_length + header_length + prelude_crc)
 _PRELUDE_LENGTH = 12
 _MAX_HEADERS_LENGTH = 128 * 1024  # 128 Kb
-_MAX_PAYLOAD_LENGTH = 16 * 1024**2  # 16 Mb
 
 
 class ParserError(Exception):
@@ -46,10 +45,14 @@ def __init__(self, length):
 
 
 class InvalidPayloadLength(ParserError):
-    """Payload length is longer than the maximum."""
+    """Payload length is longer than the maximum.
+
+    DEPRECATED: This case is no longer validated client side. Payloads
+    of varying lengths are now supported by AWS services.
+    """
 
     def __init__(self, length):
-        message = f'Payload length of {length} exceeded the maximum of {_MAX_PAYLOAD_LENGTH}'
+        message = f'Payload length of {length} exceeded the maximum of 24MB.'
         super().__init__(message)
 
 
@@ -459,9 +462,6 @@ def _validate_prelude(self, prelude):
         if prelude.headers_length > _MAX_HEADERS_LENGTH:
             raise InvalidHeadersLength(prelude.headers_length)
 
-        if prelude.payload_length > _MAX_PAYLOAD_LENGTH:
-            raise InvalidPayloadLength(prelude.payload_length)
-
     def _parse_prelude(self):
         prelude_bytes = self._data[:_PRELUDE_LENGTH]
         raw_prelude, _ = DecodeUtils.unpack_prelude(prelude_bytes)

tests/unit/test_eventstream.py

@@ -23,7 +23,6 @@
     EventStreamHeaderParser,
     EventStreamMessage,
     InvalidHeadersLength,
-    InvalidPayloadLength,
     MessagePrelude,
     NoInitialResponseError,
 )
@@ -200,6 +199,26 @@
     ),
 )
 
+# Validate payloads larger than previous 16MB limit
+# work as intended.
+EXTENDED_PAYLOAD_LENGTH = (
+    b"\x01\x00\x00\x11"  # total length
+    + b"\x00\x00\x00\x00"  # headers length
+    + b"\xf4\x08\x61\xc5"  # prelude crc
+    + b"0" * (16 * 1024**2 + 1)  # payload
+    + b"\x2a\xb4\xc5\xa5",  # message crc
+    EventStreamMessage(
+        prelude=MessagePrelude(
+            total_length=0x01000011,
+            headers_length=0x00,
+            crc=0xF40861C5,
+        ),
+        headers={},
+        payload=b"0" * (16 * 1024**2 + 1),
+        crc=0x2AB4C5A5,
+    ),
+)
+
 # Tuples of encoded messages and their expected decoded output
 POSITIVE_CASES = [
     EMPTY_MESSAGE,
@@ -211,6 +230,7 @@
     PAYLOAD_ONE_STR_HEADER,
     ALL_HEADERS_TYPES,
     ERROR_EVENT_MESSAGE,
+    EXTENDED_PAYLOAD_LENGTH,
 ]
 
 CORRUPTED_HEADER_LENGTH = (
@@ -261,17 +281,6 @@
     InvalidHeadersLength,
 )
 
-# In contrast to the CORRUPTED_PAYLOAD case, this message is otherwise
-# well-formed - the checksums match.
-INVALID_PAYLOAD_LENGTH = (
-    b"\x01\x00\x00\x11"  # total length
-    + b"\x00\x00\x00\x00"  # headers length
-    + b"\xf4\x08\x61\xc5"  # prelude crc
-    + b"0" * (16 * 1024**2 + 1)  # payload
-    + b"\x2a\xb4\xc5\xa5",  # message crc
-    InvalidPayloadLength,
-)
-
 # Tuples of encoded messages and their expected exception
 NEGATIVE_CASES = [
     CORRUPTED_LENGTH,
@@ -280,7 +289,6 @@
     CORRUPTED_HEADER_LENGTH,
     DUPLICATE_HEADER,
     INVALID_HEADERS_LENGTH,
-    INVALID_PAYLOAD_LENGTH,
 ]
 
 
@@ -348,7 +356,6 @@ def test_all_positive_cases():
         "corrupted-headers-length",
         "duplicate-headers",
         "invalid-headers-length",
-        "invalid-payload-length",
     ],
 )
 def test_negative_cases(encoded, exception):