From 07031156c13bc9d7fa01acca0d383c13e822a7ca Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Tue, 26 Jun 2018 21:01:35 +0100 Subject: [PATCH 1/7] fix for Python3.7 --- CHANGELOG.md | 9 +++++++++ lomond/_version.py | 2 +- lomond/compression.py | 18 +++++++++--------- lomond/events.py | 30 +++++++++++++++++++++++++++--- lomond/session.py | 4 ++++ lomond/stream.py | 19 +++++++++++-------- lomond/websocket.py | 19 +++++++++++++++---- tests/test_events.py | 3 ++- tests/test_integration.py | 34 +++++++++++++++++++++++++++++++++- tests/test_session.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_websocket.py | 3 +++ tox.ini | 2 +- 12 files changed, 149 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e209651..bf6edea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [0.3.1] - Unreleased + +### Fixed + +- Python3.7 compatibility + +### Added + +- ProtocolError event ## [0.3.0] - 2018-06-25 diff --git a/lomond/_version.py b/lomond/_version.py index 16852ca..8a5ea55 100644 --- a/lomond/_version.py +++ b/lomond/_version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/lomond/compression.py b/lomond/compression.py index d62db44..a9823d5 100644 --- a/lomond/compression.py +++ b/lomond/compression.py @@ -29,7 +29,9 @@ def __repr__(self): def reset_compressor(self): """Reset the compressor for the next frame.""" self._compressobj = zlib.compressobj( - 6, zlib.DEFLATED, -max(9, self.compress_wbits) + zlib.Z_DEFAULT_COMPRESSION, + zlib.DEFLATED, + -max(9, self.compress_wbits) ) def reset_decompressor(self): @@ -64,17 +66,15 @@ def get_wbits(cls, options, key): def decompress(self, frames): """Decompress payload, returned decompressed data.""" - data = b"".join( - self._decompressobj.decompress( - frame.payload + b"\x00\x00\xff\xff" - if frame.fin - else frame.payload - ) + data = [ + self._decompressobj.decompress(frame.payload) for frame in frames - ) + ] + data.append(self._decompressobj.decompress(b"\x00\x00\xff\xff")) + payload = b''.join(data) if self.reset_decompress: self.reset_decompressor() - return data + return payload def compress(self, payload): """Compress payload, return compressed data.""" diff --git a/lomond/events.py b/lomond/events.py index 3364375..f68f382 100644 --- a/lomond/events.py +++ b/lomond/events.py @@ -140,11 +140,10 @@ class Ready(Event): and successfully negotiated the websocket upgrade. :param response: A :class:`~lomond.response.Response` object. - :param str protocol: A websocket protocol or `None` if no protocol + :param str protocol: A websocket protocol or ``None`` if no protocol was supplied. :param set extensions: A set of negotiated websocket extensions. - Currently Lomond does not support any extensions, so this will - be an empty set. + Currently only the ``'permessage-deflate'`` extension is supported. """ __slots__ = ['response', 'protocol', 'extensions'] @@ -165,6 +164,31 @@ def __repr__(self): ) +class ProtocolError(Event): + """Generated when the server deviates from the protocol. + + :param str error: A description of the error. + :param bool critical: Indicates if the error is considered + 'critical'. If ``True``, Lomond will disconnect immediately. + If ``False``, Lomond will send a close message to the server. + + """ + __slots__ = ['error', 'critical'] + name = 'protocol_error' + + def __init__(self, error, critical): + self.error = error + self.critical = critical + super(ProtocolError, self).__init__() + + def __repr__(self): + return "{}(error='{}', critical={!r})".format( + self.__class__.__name__, + self.error, + self.critical + ) + + class Unresponsive(Event): """The server has not responding to pings within `ping_timeout` seconds. diff --git a/lomond/session.py b/lomond/session.py index 0df899b..04ff64f 100644 --- a/lomond/session.py +++ b/lomond/session.py @@ -68,6 +68,10 @@ def close(self): self._close_socket() self._sock = None + def force_disconnect(self): + """Force the socket to disconnect.""" + raise _ForceDisconnect() + def write(self, data): """Send raw data.""" with self._lock: diff --git a/lomond/stream.py b/lomond/stream.py index b972d62..2a33a51 100644 --- a/lomond/stream.py +++ b/lomond/stream.py @@ -32,18 +32,12 @@ def __init__(self): self.frame_parser = ClientFrameParser() self._parsed_response = False self._frames = [] - self._compression = None self._decompress = None def set_compression(self, compression): """Set a compression object for decompressing messages.""" self.frame_parser.enable_compression() - self._compression = compression - self._decompress = ( - self._compression.decompress - if self._compression - else None - ) + self._decompress = compression.decompress if compression else None def build_message(self, frames): """Return a message, built from a list of frames.""" @@ -56,7 +50,14 @@ def feed(self, data): iter_frames = iter(self.frame_parser.feed(data)) if not self._parsed_response: - header_data = next(iter_frames) + try: + header_data = next(iter_frames) + except StopIteration: + return + except ParseError as error: + raise errors.CriticalProtocolError( + text_type(error) + ) yield Response(header_data) self._parsed_response = True @@ -64,6 +65,8 @@ def feed(self, data): while True: try: frame = next(iter_frames) + except StopIteration: + return except ParseError as error: raise errors.CriticalProtocolError( text_type(error) diff --git a/lomond/websocket.py b/lomond/websocket.py index f6eb023..f3045a5 100644 --- a/lomond/websocket.py +++ b/lomond/websocket.py @@ -267,6 +267,11 @@ def _on_close(self, message): self.close(message.code, message.reason) self.state.closing = True + def force_disconnect(self): + """Force the socket to disconnect.""" + if self.state.session is not None: + self.state.session.force_disconnect() + def on_disconnect(self): """Called on disconnect.""" if self.state.session is not None: @@ -316,14 +321,16 @@ def feed(self, data): # An error that warrants an immediate disconnect. # Usually invalid unicode. log.debug('critical protocol error; %s', error) - self.on_disconnect() + yield events.ProtocolError(six.text_type(error), True) + self.force_disconnect() except errors.ProtocolError as error: # A violation of the protocol that allows for a graceful # disconnect. log.debug('protocol error; %s', error) + yield events.ProtocolError(six.text_type(error), False) self.close(Status.PROTOCOL_ERROR, six.text_type(error)) - self.on_disconnect() + self.force_disconnect() except GeneratorExit: # The generator has exited prematurely, due to an exception @@ -402,19 +409,23 @@ def on_response(self, response): ) protocol = response.get('sec-websocket-protocol') - extensions = set(response.get_list('sec-websocket-extensions')) - self.process_extensions(extensions) + extensions = self.process_extensions( + response.get_list('sec-websocket-extensions') + ) return protocol, extensions def process_extensions(self, extensions): """Process extension headers.""" + extensions = set() for extension in extensions: extension_token, options = parse_extension(extension) if extension_token == 'permessage-deflate': + extensions.add('permessage-deflate') compression = Deflate.from_options(options) self.state.compression = compression self.state.stream.set_compression(compression) log.debug('%r enabled', compression) + return extensions def send_ping(self, data=b''): """Send a ping packet. diff --git a/tests/test_events.py b/tests/test_events.py index 27d8999..7723e70 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -35,7 +35,8 @@ (events.UnknownMessage('?.!'), "UnknownMessage()"), (events.Ping('o |'), "Ping(data='o |')"), (events.Pong(' | o'), "Pong(data=' | o')"), - (events.BackOff(0.1), "BackOff(delay=0.1)") + (events.BackOff(0.1), "BackOff(delay=0.1)"), + (events.ProtocolError('error', critical=False), "ProtocolError(error='error', critical=False)") ] # we are splitting these two test cases into separate branches, because the diff --git a/tests/test_integration.py b/tests/test_integration.py index 2ed3c57..ee4045b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -42,6 +42,21 @@ def open(self): yield self.stream.close() +class BrokenHandler(websocket.WebSocketHandler): + """Writes text/binary then closes the socket.""" + + def check_origin(self, origin): + return True + + @gen.coroutine + def open(self): + self.set_nodelay(True) + yield self.write_message(u'foo') + yield self.stream.write(b'WebSocket? WTF is a WebSocket?') + yield self.write_message(b'bar', binary=True) + yield self.stream.close() + + class EchoHandler(websocket.WebSocketHandler): """Echos any message sent to it.""" @@ -69,7 +84,8 @@ def run_server(cls, port=8080): app = web.Application([ (r'^/graceful$', GracefulHandler), (r'^/non-graceful$', NonGracefulHandler), - (r'^/echo$', EchoHandler) + (r'^/echo$', EchoHandler), + (r'^/broken', BrokenHandler) ]) cls.server = server = httpserver.HTTPServer(app) cls.loop = ioloop.IOLoop.current() @@ -123,6 +139,22 @@ def test_non_graceful(self): assert events[6].name == 'disconnected' assert not events[6].graceful + def test_broken(self): + """Test server that closes gracefully.""" + ws = WebSocket(self.WS_URL + 'broken') + events = list(ws.connect(ping_rate=0)) + assert len(events) == 7 + assert events[0].name == 'connecting' + assert events[1].name == 'connected' + assert events[2].name == 'ready' + assert events[3].name == 'poll' + assert events[4].name == 'text' + assert events[4].text == u'foo' + assert events[5].name == 'protocol_error' + assert not events[5].critical + assert events[6].name == 'disconnected' + assert not events[6].graceful + def test_echo(self): """Test echo server.""" ws = WebSocket(self.WS_URL + 'echo') diff --git a/tests/test_session.py b/tests/test_session.py index ed774d5..f87b669 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -337,6 +337,40 @@ def test_simple_run(monkeypatch, mocker): assert not _events[5].graceful + +@mocketize +def test_simple_run_error(monkeypatch, mocker): + monkeypatch.setattr( + 'os.urandom', b'\x00'.__mul__ + ) + # Header is too large + Mocket.register( + MocketEntry( + ('example.com', 80), + [b'X' * 32768] + ) + ) + + # mocket doesn't support .pending() call which is used when ssl is used + session = WebsocketSession(WebSocket('ws://example.com/')) + session._selector_cls = FakeSelector + session._on_ready() + session._regular_orig = session._regular + + mocker.patch( + 'lomond.websocket.WebSocket._send_close') + mocker.patch.object(session.websocket, 'send_ping') + + _events = list(session.run()) + assert len(_events) == 4 + assert isinstance(_events[0], events.Connecting) + assert isinstance(_events[1], events.Connected) + assert isinstance(_events[2], events.ProtocolError) + assert _events[2].critical + assert isinstance(_events[3], events.Disconnected) + assert not _events[3].graceful + + @mocketize def test_simple_run_with_close(monkeypatch, mocker): """Test graceful close.""" diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 84084d9..4cf8ded 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -34,6 +34,9 @@ def session_time(self): def close(self): pass + def force_disconnect(self): + pass + @pytest.fixture def websocket(monkeypatch): diff --git a/tox.ini b/tox.ini index 82ba67d..41a7c16 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,py{27,35,36}{,-wsaccel},coverage +envlist = clean,py{27,35,36,37}{,-wsaccel},coverage [testenv:clean] deps = coverage From ce2cd459978bb8d64487bb09d86af8549469a6e8 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 10:06:54 +0100 Subject: [PATCH 2/7] fix --- lomond/websocket.py | 6 +++--- tests/test_integration.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lomond/websocket.py b/lomond/websocket.py index f3045a5..4b7f6e1 100644 --- a/lomond/websocket.py +++ b/lomond/websocket.py @@ -416,16 +416,16 @@ def on_response(self, response): def process_extensions(self, extensions): """Process extension headers.""" - extensions = set() + enabled_extensions = set() for extension in extensions: extension_token, options = parse_extension(extension) if extension_token == 'permessage-deflate': - extensions.add('permessage-deflate') + enabled_extensions.add('permessage-deflate') compression = Deflate.from_options(options) self.state.compression = compression self.state.stream.set_compression(compression) log.debug('%r enabled', compression) - return extensions + return enabled_extensions def send_ping(self, data=b''): """Send a ping packet. diff --git a/tests/test_integration.py b/tests/test_integration.py index ee4045b..8edb17f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -186,6 +186,7 @@ def test_echo_compress(self): events = [] for event in ws.connect(poll=60, ping_rate=0, auto_pong=False): events.append(event) + print(event) if event.name == 'ready': assert ws.supports_compression ws.send_text(u'echofoo') From ca5fd3fa5b725fad1f30e2eff1a311a09e5aff46 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 10:09:09 +0100 Subject: [PATCH 3/7] debug --- tests/test_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 8edb17f..ee4045b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -186,7 +186,6 @@ def test_echo_compress(self): events = [] for event in ws.connect(poll=60, ping_rate=0, auto_pong=False): events.append(event) - print(event) if event.name == 'ready': assert ws.supports_compression ws.send_text(u'echofoo') From 93e0e71c1f79ac2644d5f18bd659d6f4c841ccf9 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 10:14:23 +0100 Subject: [PATCH 4/7] Circle doesn't do py37 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 41a7c16..82ba67d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,py{27,35,36,37}{,-wsaccel},coverage +envlist = clean,py{27,35,36}{,-wsaccel},coverage [testenv:clean] deps = coverage From a0250a9a3d89fa1a664e897072dc1c4ae497ddb0 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 11:34:38 +0100 Subject: [PATCH 5/7] Python3.7 in tests --- circle.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 86617ae..ff887ac 100644 --- a/circle.yml +++ b/circle.yml @@ -1,4 +1,4 @@ dependencies: override: - pip install tox tox-pyenv - - pyenv local 2.7.11 3.5.2 3.6.2 + - pyenv local 2.7.11 3.5.2 3.6.2 3.7.0rc1 diff --git a/tox.ini b/tox.ini index 82ba67d..41a7c16 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,py{27,35,36}{,-wsaccel},coverage +envlist = clean,py{27,35,36,37}{,-wsaccel},coverage [testenv:clean] deps = coverage From c74b61a9cd664943bb14221d2815ac201207106e Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 11:47:41 +0100 Subject: [PATCH 6/7] CI doesn't support Py3.7 after all --- circle.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index ff887ac..86617ae 100644 --- a/circle.yml +++ b/circle.yml @@ -1,4 +1,4 @@ dependencies: override: - pip install tox tox-pyenv - - pyenv local 2.7.11 3.5.2 3.6.2 3.7.0rc1 + - pyenv local 2.7.11 3.5.2 3.6.2 diff --git a/tox.ini b/tox.ini index 41a7c16..82ba67d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,py{27,35,36,37}{,-wsaccel},coverage +envlist = clean,py{27,35,36}{,-wsaccel},coverage [testenv:clean] deps = coverage From 538bf99f8cda92db99a7e8766d1762c299bd50bd Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 27 Jun 2018 13:39:38 +0100 Subject: [PATCH 7/7] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6edea..14e6384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [0.3.1] - Unreleased +## [0.3.1] - 2018-06-27 ### Fixed