From 8513fe82c586530e5fc361154ef974702e3ef116 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 17 Jun 2022 15:25:11 -0300 Subject: [PATCH 01/54] Upgrade pyperf (drop support for Python 2.x) Since 0.8.1 pyte does not support Python 2.x anymore so it makes sense to upgrade one of its dev dependencies, pyperf. --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index c1ee84d..8c54e74 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ pytest -pyperf == 1.7.1 +pyperf >= 2.3.0 wcwidth wheel From cabc0a5bb14bf19e9e4924789f691d1813ede5d3 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 17 Jun 2022 15:55:38 -0300 Subject: [PATCH 02/54] Allow change the screen geometry Receive via environ the geometry of the screen to test with a default of 24 lines by 80 columns. Add this and the input file into Runner's metadata so it is preserved in the log file (if any) --- benchmark.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/benchmark.py b/benchmark.py index 75657db..4500c83 100644 --- a/benchmark.py +++ b/benchmark.py @@ -10,6 +10,11 @@ ..................... ls.input: Mean +- std dev: 644 ns +- 23 ns + Environment variables: + + BENCHMARK: the input file to feed pyte's Stream and render on the Screen + GEOMETRY: the dimensions of the screen with format "x" (default 24x80) + :copyright: (c) 2016-2021 by pyte authors and contributors, see AUTHORS for details. :license: LGPL, see LICENSE for more details. @@ -28,20 +33,25 @@ import pyte -def make_benchmark(path, screen_cls): +def make_benchmark(path, screen_cls, columns, lines): with io.open(path, "rt", encoding="utf-8") as handle: data = handle.read() - stream = pyte.Stream(screen_cls(80, 24)) + stream = pyte.Stream(screen_cls(columns, lines)) return partial(stream.feed, data) if __name__ == "__main__": benchmark = os.environ["BENCHMARK"] - sys.argv.extend(["--inherit-environ", "BENCHMARK"]) + lines, columns = map(int, os.environ.get("GEOMETRY", "24x80").split('x')) + sys.argv.extend(["--inherit-environ", "BENCHMARK,GEOMETRY"]) - runner = Runner() + runner = Runner(metadata={ + 'input_file': benchmark, + 'columns': columns, + 'lines': lines + }) for screen_cls in [pyte.Screen, pyte.DiffScreen, pyte.HistoryScreen]: name = os.path.basename(benchmark) + "->" + screen_cls.__name__ - runner.bench_func(name, make_benchmark(benchmark, screen_cls)) + runner.bench_func(name, make_benchmark(benchmark, screen_cls, columns, lines)) From 940e19b4b6420f3007cfcf40327e384e791d1d10 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 17 Jun 2022 19:04:17 -0300 Subject: [PATCH 03/54] Impl benchmark tests for screen.display, .reset and .resize Implement three more benchmark scenarios for testing screen.display, screen.reset and screen.resize. For the standard 24x80 geometry, these methods have a negligible cost however of larger geometries, they can be up to 100 times slower than stream.feed so benchmarking them is important. Changed how the metadata is stored so on each bench_func call we encode which scenario are we testing, with which screen class and geometry. --- benchmark.py | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/benchmark.py b/benchmark.py index 4500c83..438c241 100644 --- a/benchmark.py +++ b/benchmark.py @@ -10,6 +10,10 @@ ..................... ls.input: Mean +- std dev: 644 ns +- 23 ns + $ BENCHMARK=tests/captured/ls.input GEOMETRY=1024x1024 python benchmark.py -o results.json + ..................... + ls.input: Mean +- std dev: 644 ns +- 23 ns + Environment variables: BENCHMARK: the input file to feed pyte's Stream and render on the Screen @@ -32,26 +36,54 @@ import pyte - -def make_benchmark(path, screen_cls, columns, lines): +def setup(path, screen_cls, columns, lines): with io.open(path, "rt", encoding="utf-8") as handle: data = handle.read() - stream = pyte.Stream(screen_cls(columns, lines)) + screen = screen_cls(columns, lines) + stream = pyte.Stream(screen) + + return data, screen, stream + +def make_stream_feed_benchmark(path, screen_cls, columns, lines): + data, _, stream = setup(path, screen_cls, columns, lines) return partial(stream.feed, data) +def make_screen_display_benchmark(path, screen_cls, columns, lines): + data, screen, stream = setup(path, screen_cls, columns, lines) + stream.feed(data) + return lambda: screen.display + +def make_screen_reset_benchmark(path, screen_cls, columns, lines): + data, screen, stream = setup(path, screen_cls, columns, lines) + stream.feed(data) + return screen.reset + +def make_screen_resize_half_benchmark(path, screen_cls, columns, lines): + data, screen, stream = setup(path, screen_cls, columns, lines) + stream.feed(data) + return partial(screen.resize, lines=lines//2, columns=columns//2) if __name__ == "__main__": benchmark = os.environ["BENCHMARK"] lines, columns = map(int, os.environ.get("GEOMETRY", "24x80").split('x')) sys.argv.extend(["--inherit-environ", "BENCHMARK,GEOMETRY"]) - runner = Runner(metadata={ + runner = Runner() + + metadata = { 'input_file': benchmark, 'columns': columns, 'lines': lines - }) + } + benchmark_name = os.path.basename(benchmark) for screen_cls in [pyte.Screen, pyte.DiffScreen, pyte.HistoryScreen]: - name = os.path.basename(benchmark) + "->" + screen_cls.__name__ - runner.bench_func(name, make_benchmark(benchmark, screen_cls, columns, lines)) + screen_cls_name = screen_cls.__name__ + for make_test in (make_stream_feed_benchmark, make_screen_display_benchmark, make_screen_reset_benchmark, make_screen_resize_half_benchmark): + scenario = make_test.__name__[5:-10] # remove make_ and _benchmark + + name = f"[{scenario} {lines}x{columns}] {benchmark_name}->{screen_cls_name}" + metadata.update({'scenario': scenario, 'screen_cls': screen_cls_name}) + runner.bench_func(name, make_test(benchmark, screen_cls, columns, lines), metadata=metadata) + From 0b8007a8e574bcda7e50f039fc72cd77327d3a81 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 18 Jun 2022 09:04:21 -0300 Subject: [PATCH 04/54] Impl script to run a full benchmark A shell script to test all the captured input files and run them under different terminal geometries (24x80, 240x800, 2400x8000, 24x8000 and 2400x80). These settings aim to stress pyte with larger and larger screens (by a 10 factor on both dimensions and on each dimension separately). --- full_benchmark.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 full_benchmark.sh diff --git a/full_benchmark.sh b/full_benchmark.sh new file mode 100755 index 0000000..a3fe567 --- /dev/null +++ b/full_benchmark.sh @@ -0,0 +1,39 @@ +#!/usr/bin/bash + +if [ "$#" != "1" ]; then + echo "Usage benchmark.sh " + exit 1 +fi +outputfile=$1 + +if [ ! -f benchmark.py ]; then + echo "File benchmark.py missing. Are you in the home folder of pyte project?" + exit 1 +fi + +for inputfile in $(ls -1 tests/captured/*.input); do + export GEOMETRY=24x80 + echo "$inputfile - $GEOMETRY" + echo "======================" + BENCHMARK=$inputfile python benchmark.py --append $outputfile + + export GEOMETRY=240x800 + echo "$inputfile - $GEOMETRY" + echo "======================" + BENCHMARK=$inputfile python benchmark.py --append $outputfile + + export GEOMETRY=2400x8000 + echo "$inputfile - $GEOMETRY" + echo "======================" + BENCHMARK=$inputfile python benchmark.py --append $outputfile + + export GEOMETRY=24x8000 + echo "$inputfile - $GEOMETRY" + echo "======================" + BENCHMARK=$inputfile python benchmark.py --append $outputfile + + export GEOMETRY=2400x80 + echo "$inputfile - $GEOMETRY" + echo "======================" + BENCHMARK=$inputfile python benchmark.py --append $outputfile +done From e0b0e8be6e634296711cf1ee39610feb8bac0f29 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 2 Jul 2022 18:47:01 -0300 Subject: [PATCH 05/54] Fix benchmark.py using ByteStream and not Stream The input files in the tests/captured must be loaded with ByteStream and not Stream, otherwise the \r are lost and the benchmark results may not reflect real scenarios. --- benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark.py b/benchmark.py index 438c241..b52787e 100644 --- a/benchmark.py +++ b/benchmark.py @@ -37,11 +37,11 @@ import pyte def setup(path, screen_cls, columns, lines): - with io.open(path, "rt", encoding="utf-8") as handle: + with io.open(path, "rb") as handle: data = handle.read() screen = screen_cls(columns, lines) - stream = pyte.Stream(screen) + stream = pyte.ByteStream(screen) return data, screen, stream From eec4a2ee0afbb31132c0cba689d49bf068533ddc Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 14:03:04 -0300 Subject: [PATCH 06/54] Enable optionally tracemalloc on full benchmark --- full_benchmark.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/full_benchmark.sh b/full_benchmark.sh index a3fe567..02a88dd 100755 --- a/full_benchmark.sh +++ b/full_benchmark.sh @@ -1,9 +1,21 @@ #!/usr/bin/bash -if [ "$#" != "1" ]; then +if [ "$#" != "1" -a "$#" != "2" ]; then echo "Usage benchmark.sh " + echo "Usage benchmark.sh tracemalloc" exit 1 fi + +if [ "$2" = "tracemalloc" ]; then + tracemalloc="--tracemalloc" +elif [ "$2" = "" ]; then + tracemalloc="" +else + echo "Usage benchmark.sh " + echo "Usage benchmark.sh tracemalloc" + exit 1 +fi + outputfile=$1 if [ ! -f benchmark.py ]; then @@ -15,25 +27,25 @@ for inputfile in $(ls -1 tests/captured/*.input); do export GEOMETRY=24x80 echo "$inputfile - $GEOMETRY" echo "======================" - BENCHMARK=$inputfile python benchmark.py --append $outputfile + BENCHMARK=$inputfile python benchmark.py $tracemalloc --append $outputfile export GEOMETRY=240x800 echo "$inputfile - $GEOMETRY" echo "======================" - BENCHMARK=$inputfile python benchmark.py --append $outputfile + BENCHMARK=$inputfile python benchmark.py $tracemalloc --append $outputfile export GEOMETRY=2400x8000 echo "$inputfile - $GEOMETRY" echo "======================" - BENCHMARK=$inputfile python benchmark.py --append $outputfile + BENCHMARK=$inputfile python benchmark.py $tracemalloc --append $outputfile export GEOMETRY=24x8000 echo "$inputfile - $GEOMETRY" echo "======================" - BENCHMARK=$inputfile python benchmark.py --append $outputfile + BENCHMARK=$inputfile python benchmark.py $tracemalloc --append $outputfile export GEOMETRY=2400x80 echo "$inputfile - $GEOMETRY" echo "======================" - BENCHMARK=$inputfile python benchmark.py --append $outputfile + BENCHMARK=$inputfile python benchmark.py $tracemalloc --append $outputfile done From f899535989927b378c7bb40d543f13b5359514c5 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 18 Jun 2022 15:31:00 -0300 Subject: [PATCH 07/54] display meth: iterate over data entries filling the gap between The former `for x in range(...)` implementation iterated over the all the possibly indexes (for columns and lines) wasting cyclies because some of those indexes (and in some cases most) pointed to non-existing entries. These non-existing entries were faked and a default character was returned in place. This commit instead makes display to iterate over the existing entries. When gaps between to entries are detected, the gap is filled with the same default character without having to pay for indexing non-entries. Note: I found that in the current implementation of screen, screen.buffer may have entries (chars in a line) outside of the width of the screen. At the display method those are filtered out however I'm not sure if this is not a real bug that was uncovered because never we iterated over the data entries. If this is true, we may be wasting space as we keep in memory chars that are outside of the screen. --- pyte/screens.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 5e7f759..e6bdfe2 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -228,18 +228,48 @@ def __repr__(self): @property def display(self): """A :func:`list` of screen lines as unicode strings.""" + padding = self.default_char.data + def render(line): is_wide_char = False - for x in range(self.columns): + prev_x = -1 + for x, cell in sorted(line.items()): + # TODO apparently a line can hold more items (chars) than + # suppose to (outside of the range of 0-cols). + if x >= self.columns: + break + + gap = x - (prev_x + 1) + if gap: + yield padding * gap + + prev_x = x + if is_wide_char: # Skip stub is_wide_char = False continue - char = line[x].data + char = cell.data assert sum(map(wcwidth, char[1:])) == 0 is_wide_char = wcwidth(char[0]) == 2 yield char - return ["".join(render(self.buffer[y])) for y in range(self.lines)] + gap = self.columns - (prev_x + 1) + if gap: + yield padding * gap + + prev_y = -1 + output = [] + for y, line in sorted(self.buffer.items()): + empty_lines = y - (prev_y + 1) + output.extend([padding * self.columns] * empty_lines) + prev_y = y + + output.append("".join(render(line))) + + empty_lines = self.lines - (prev_y + 1) + output.extend([padding * self.columns] * empty_lines) + + return output def reset(self): """Reset the terminal to its initial state. From b3b7db4cd45a66acd40f10f601b0e211f53715e9 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 18 Jun 2022 17:33:19 -0300 Subject: [PATCH 08/54] Inline generator into display inner loop Python generators (yield) and function calls are slower then normal for-loops. Improve screen.display by x1 to x1.8 times faster by inlining the code. --- pyte/screens.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index e6bdfe2..28d0acb 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -230,18 +230,25 @@ def display(self): """A :func:`list` of screen lines as unicode strings.""" padding = self.default_char.data - def render(line): + prev_y = -1 + output = [] + columns = self.columns + for y, line in sorted(self.buffer.items()): + empty_lines = y - (prev_y + 1) + if empty_lines: + output.extend([padding * columns] * empty_lines) + prev_y = y + is_wide_char = False prev_x = -1 + display_line = [] for x, cell in sorted(line.items()): - # TODO apparently a line can hold more items (chars) than - # suppose to (outside of the range of 0-cols). - if x >= self.columns: + if x >= columns: break gap = x - (prev_x + 1) if gap: - yield padding * gap + display_line.append(padding * gap) prev_x = x @@ -251,23 +258,17 @@ def render(line): char = cell.data assert sum(map(wcwidth, char[1:])) == 0 is_wide_char = wcwidth(char[0]) == 2 - yield char + display_line.append(char) - gap = self.columns - (prev_x + 1) + gap = columns - (prev_x + 1) if gap: - yield padding * gap - - prev_y = -1 - output = [] - for y, line in sorted(self.buffer.items()): - empty_lines = y - (prev_y + 1) - output.extend([padding * self.columns] * empty_lines) - prev_y = y + display_line.append(padding * gap) - output.append("".join(render(line))) + output.append("".join(display_line)) empty_lines = self.lines - (prev_y + 1) - output.extend([padding * self.columns] * empty_lines) + if empty_lines: + output.extend([padding * columns] * empty_lines) return output From de592450ee60d07356cce9cd5091b74ef8cb17e2 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Mon, 20 Jun 2022 10:37:09 -0300 Subject: [PATCH 09/54] Move assert out of prod code The assert that checks the width of each char is removed from screen.display and put it into the tests. This ensures that our test suite maintains the same quality and at the same time we make screen.display ~x1.7 faster. --- pyte/screens.py | 1 - tests/helpers/asserts.py | 12 +++++++ tests/test_history.py | 28 ++++++++++++++- tests/test_input_output.py | 5 ++- tests/test_screen.py | 70 +++++++++++++++++++++++++++++++++++++- tests/test_stream.py | 7 +++- 6 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 tests/helpers/asserts.py diff --git a/pyte/screens.py b/pyte/screens.py index 28d0acb..cf122ae 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -256,7 +256,6 @@ def display(self): is_wide_char = False continue char = cell.data - assert sum(map(wcwidth, char[1:])) == 0 is_wide_char = wcwidth(char[0]) == 2 display_line.append(char) diff --git a/tests/helpers/asserts.py b/tests/helpers/asserts.py new file mode 100644 index 0000000..5ab2689 --- /dev/null +++ b/tests/helpers/asserts.py @@ -0,0 +1,12 @@ +from wcwidth import wcwidth +def consistency_asserts(screen): + # Ensure that all the cells in the buffer, if they have + # a data of 2 or more code points, they all sum up 0 width + # In other words, the width of the cell is determinated by the + # width of the first code point. + for y in range(screen.lines): + for x in range(screen.columns): + char = screen.buffer[y][x].data + assert sum(map(wcwidth, char[1:])) == 0 + + diff --git a/tests/test_history.py b/tests/test_history.py index 52084ec..288b940 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1,8 +1,10 @@ -import os +import os, sys import pyte from pyte import control as ctrl, modes as mo +sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) +from asserts import consistency_asserts def chars(history_lines, columns): return ["".join(history_lines[y][x].data for x in range(columns)) @@ -96,6 +98,7 @@ def test_prev_page(): "39 ", " " ] + consistency_asserts(screen) assert chars(screen.history.top, screen.columns)[-4:] == [ "33 ", @@ -114,6 +117,7 @@ def test_prev_page(): "37 ", "38 " ] + consistency_asserts(screen) assert chars(screen.history.top, screen.columns)[-4:] == [ "31 ", @@ -138,6 +142,7 @@ def test_prev_page(): "35 ", "36 ", ] + consistency_asserts(screen) assert len(screen.history.bottom) == 4 assert chars(screen.history.bottom, screen.columns) == [ @@ -165,6 +170,7 @@ def test_prev_page(): "49 ", " " ] + consistency_asserts(screen) screen.prev_page() assert screen.history.position == 47 @@ -175,6 +181,7 @@ def test_prev_page(): "46 ", "47 " ] + consistency_asserts(screen) assert len(screen.history.bottom) == 3 assert chars(screen.history.bottom, screen.columns) == [ @@ -200,6 +207,7 @@ def test_prev_page(): "39 ", " " ] + consistency_asserts(screen) screen.prev_page() assert screen.history.position == 37 @@ -209,6 +217,7 @@ def test_prev_page(): "36 ", "37 " ] + consistency_asserts(screen) assert len(screen.history.bottom) == 3 assert chars(screen.history.bottom, screen.columns) == [ @@ -235,6 +244,7 @@ def test_prev_page(): "49 ", " " ] + consistency_asserts(screen) screen.cursor_to_line(screen.lines // 2) @@ -250,6 +260,7 @@ def test_prev_page(): "4 ", "5 " ] + consistency_asserts(screen) while screen.history.position < screen.history.size: screen.next_page() @@ -262,6 +273,7 @@ def test_prev_page(): "49 ", " " ] + consistency_asserts(screen) # e) same with cursor near the middle of the screen. screen = pyte.HistoryScreen(5, 5, history=50) @@ -282,6 +294,7 @@ def test_prev_page(): "49 ", " " ] + consistency_asserts(screen) screen.cursor_to_line(screen.lines // 2 - 2) @@ -297,6 +310,7 @@ def test_prev_page(): "4 ", "5 " ] + consistency_asserts(screen) while screen.history.position < screen.history.size: screen.next_page() @@ -310,6 +324,7 @@ def test_prev_page(): "49 ", " " ] + consistency_asserts(screen) def test_next_page(): @@ -332,6 +347,7 @@ def test_next_page(): "24 ", " " ] + consistency_asserts(screen) # a) page up -- page down. screen.prev_page() @@ -346,6 +362,7 @@ def test_next_page(): "24 ", " " ] + consistency_asserts(screen) # b) double page up -- page down. screen.prev_page() @@ -366,6 +383,7 @@ def test_next_page(): "21 ", "22 " ] + consistency_asserts(screen) # c) double page up -- double page down screen.prev_page() @@ -381,6 +399,7 @@ def test_next_page(): "21 ", "22 " ] + consistency_asserts(screen) def test_ensure_width(monkeypatch): @@ -402,6 +421,7 @@ def test_ensure_width(monkeypatch): "0024 ", " " ] + consistency_asserts(screen) # Shrinking the screen should truncate the displayed lines following lines. screen.resize(5, 3) @@ -416,6 +436,7 @@ def test_ensure_width(monkeypatch): "002", # 21 "002" # 22 ] + consistency_asserts(screen) def test_not_enough_lines(): @@ -436,6 +457,7 @@ def test_not_enough_lines(): "4 ", " " ] + consistency_asserts(screen) screen.prev_page() assert not screen.history.top @@ -448,6 +470,7 @@ def test_not_enough_lines(): "3 ", "4 ", ] + consistency_asserts(screen) screen.next_page() assert screen.history.top @@ -459,6 +482,7 @@ def test_not_enough_lines(): "4 ", " " ] + consistency_asserts(screen) def test_draw(monkeypatch): @@ -479,6 +503,7 @@ def test_draw(monkeypatch): "24 ", " " ] + consistency_asserts(screen) # a) doing a pageup and then a draw -- expecting the screen # to scroll to the bottom before drawing anything. @@ -494,6 +519,7 @@ def test_draw(monkeypatch): "24 ", "x " ] + consistency_asserts(screen) def test_cursor_is_hidden(monkeypatch): diff --git a/tests/test_input_output.py b/tests/test_input_output.py index 4d25c03..ab6535d 100644 --- a/tests/test_input_output.py +++ b/tests/test_input_output.py @@ -1,5 +1,5 @@ import json -import os.path +import os.path, sys import pytest @@ -8,6 +8,8 @@ captured_dir = os.path.join(os.path.dirname(__file__), "captured") +sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) +from asserts import consistency_asserts @pytest.mark.parametrize("name", [ "cat-gpl3", "find-etc", "htop", "ls", "mc", "top", "vi" @@ -23,3 +25,4 @@ def test_input_output(name): stream = pyte.ByteStream(screen) stream.feed(input) assert screen.display == output + consistency_asserts(screen) diff --git a/tests/test_screen.py b/tests/test_screen.py index b6ba90d..b4fe75a 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1,4 +1,4 @@ -import copy +import copy, sys, os import pytest @@ -6,6 +6,8 @@ from pyte import modes as mo, control as ctrl, graphics as g from pyte.screens import Char +sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) +from asserts import consistency_asserts # Test helpers. @@ -230,12 +232,14 @@ def test_resize(): screen = update(pyte.Screen(2, 2), ["bo", "sh"], [None, None]) screen.resize(2, 3) assert screen.display == ["bo ", "sh "] + consistency_asserts(screen) # b) if the current display is wider than the requested size, # columns should be removed from the right... screen = update(pyte.Screen(2, 2), ["bo", "sh"], [None, None]) screen.resize(2, 1) assert screen.display == ["b", "s"] + consistency_asserts(screen) # c) if the current display is shorter than the requested # size, new rows should be added on the bottom. @@ -243,12 +247,14 @@ def test_resize(): screen.resize(3, 2) assert screen.display == ["bo", "sh", " "] + consistency_asserts(screen) # d) if the current display is taller than the requested # size, rows should be removed from the top. screen = update(pyte.Screen(2, 2), ["bo", "sh"], [None, None]) screen.resize(1, 2) assert screen.display == ["sh"] + consistency_asserts(screen) def test_resize_same(): @@ -312,6 +318,7 @@ def test_draw(): assert screen.display == ["abc", " ", " "] assert (screen.cursor.y, screen.cursor.x) == (0, 3) + consistency_asserts(screen) # ... one` more character -- now we got a linefeed! screen.draw("a") @@ -326,11 +333,13 @@ def test_draw(): assert screen.display == ["abc", " ", " "] assert (screen.cursor.y, screen.cursor.x) == (0, 3) + consistency_asserts(screen) # No linefeed is issued on the end of the line ... screen.draw("a") assert screen.display == ["aba", " ", " "] assert (screen.cursor.y, screen.cursor.x) == (0, 3) + consistency_asserts(screen) # ``IRM`` mode is on, expecting new characters to move the old ones # instead of replacing them. @@ -338,10 +347,12 @@ def test_draw(): screen.cursor_position() screen.draw("x") assert screen.display == ["xab", " ", " "] + consistency_asserts(screen) screen.cursor_position() screen.draw("y") assert screen.display == ["yxa", " ", " "] + consistency_asserts(screen) def test_draw_russian(): @@ -350,6 +361,7 @@ def test_draw_russian(): stream = pyte.Stream(screen) stream.feed("Нерусский текст") assert screen.display == ["Нерусский текст "] + consistency_asserts(screen) def test_draw_multiple_chars(): @@ -357,6 +369,7 @@ def test_draw_multiple_chars(): screen.draw("foobar") assert screen.cursor.x == 6 assert screen.display == ["foobar "] + consistency_asserts(screen) def test_draw_utf8(): @@ -365,6 +378,7 @@ def test_draw_utf8(): stream = pyte.ByteStream(screen) stream.feed(b"\xE2\x80\x9D") assert screen.display == ["”"] + consistency_asserts(screen) def test_draw_width2(): @@ -387,12 +401,14 @@ def test_draw_width2_irm(): screen.draw("コ") assert screen.display == ["コ"] assert tolist(screen) == [[Char("コ"), Char(" ")]] + consistency_asserts(screen) # Overwrite the stub part of a width 2 character. screen.set_mode(mo.IRM) screen.cursor_to_column(screen.columns) screen.draw("x") assert screen.display == [" x"] + consistency_asserts(screen) def test_draw_width0_combining(): @@ -401,17 +417,20 @@ def test_draw_width0_combining(): # a) no prev. character screen.draw("\N{COMBINING DIAERESIS}") assert screen.display == [" ", " "] + consistency_asserts(screen) screen.draw("bad") # b) prev. character is on the same line screen.draw("\N{COMBINING DIAERESIS}") assert screen.display == ["bad̈ ", " "] + consistency_asserts(screen) # c) prev. character is on the prev. line screen.draw("!") screen.draw("\N{COMBINING DIAERESIS}") assert screen.display == ["bad̈!̈", " "] + consistency_asserts(screen) def test_draw_width0_irm(): @@ -422,6 +441,7 @@ def test_draw_width0_irm(): screen.draw("\N{ZERO WIDTH SPACE}") screen.draw("\u0007") # DELETE. assert screen.display == [" " * screen.columns] + consistency_asserts(screen) def test_draw_width0_decawm_off(): @@ -446,6 +466,7 @@ def test_draw_cp437(): stream.feed("α ± ε".encode("cp437")) assert screen.display == ["α ± ε"] + consistency_asserts(screen) def test_draw_with_carriage_return(): @@ -465,12 +486,14 @@ def test_draw_with_carriage_return(): "pcrm sem ;ps aux|grep -P 'httpd|fcgi'|grep -v grep", "}'|xargs kill -9;/etc/init.d/httpd startssl " ] + consistency_asserts(screen) def test_display_wcwidth(): screen = pyte.Screen(10, 1) screen.draw("コンニチハ") assert screen.display == ["コンニチハ"] + consistency_asserts(screen) def test_carriage_return(): @@ -519,6 +542,7 @@ def test_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) # ... and again ... screen.index() @@ -531,6 +555,7 @@ def test_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) # ... and again ... screen.index() @@ -543,6 +568,7 @@ def test_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) # look, nothing changes! screen.index() @@ -555,6 +581,7 @@ def test_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) def test_reverse_index(): @@ -594,6 +621,7 @@ def test_reverse_index(): [Char("t", fg="red"), Char("h", fg="red")], [Char("o"), Char("h")], ] + consistency_asserts(screen) # ... and again ... screen.reverse_index() @@ -606,6 +634,7 @@ def test_reverse_index(): [Char("s"), Char("h")], [Char("o"), Char("h")], ] + consistency_asserts(screen) # ... and again ... screen.reverse_index() @@ -618,6 +647,7 @@ def test_reverse_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) # look, nothing changes! screen.reverse_index() @@ -630,6 +660,7 @@ def test_reverse_index(): [screen.default_char, screen.default_char], [Char("o"), Char("h")], ] + consistency_asserts(screen) def test_linefeed(): @@ -810,6 +841,7 @@ def test_insert_lines(): [Char("s"), Char("a"), Char("m")], [Char("i", fg="red"), Char("s", fg="red"), Char(" ", fg="red")], ] + consistency_asserts(screen) screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[1]) screen.insert_lines(2) @@ -821,6 +853,7 @@ def test_insert_lines(): [screen.default_char] * 3, [Char("s"), Char("a"), Char("m")] ] + consistency_asserts(screen) # b) with margins screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], @@ -838,6 +871,7 @@ def test_insert_lines(): [Char("f", fg="red"), Char("o", fg="red"), Char("o", fg="red")], [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], colored=[2, 3]) @@ -854,6 +888,7 @@ def test_insert_lines(): [Char("b", fg="red"), Char("a", fg="red"), Char("r", fg="red")], [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) screen.insert_lines(2) assert (screen.cursor.y, screen.cursor.x) == (1, 0) @@ -865,6 +900,7 @@ def test_insert_lines(): [Char("b", fg="red"), Char("a", fg="red"), Char("r", fg="red")], [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) # c) with margins -- trying to insert more than we have available screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], @@ -882,6 +918,7 @@ def test_insert_lines(): [screen.default_char] * 3, [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) # d) with margins -- trying to insert outside scroll boundaries; # expecting nothing to change @@ -899,6 +936,7 @@ def test_insert_lines(): [Char("b", fg="red"), Char("a", fg="red"), Char("r", fg="red")], [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) def test_delete_lines(): @@ -913,6 +951,7 @@ def test_delete_lines(): [Char("f"), Char("o"), Char("o")], [screen.default_char] * 3, ] + consistency_asserts(screen) screen.delete_lines(0) @@ -923,6 +962,7 @@ def test_delete_lines(): [screen.default_char] * 3, [screen.default_char] * 3, ] + consistency_asserts(screen) # b) with margins screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], @@ -940,6 +980,7 @@ def test_delete_lines(): [screen.default_char] * 3, [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], colored=[2, 3]) @@ -956,6 +997,7 @@ def test_delete_lines(): [screen.default_char] * 3, [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) # c) with margins -- trying to delete more than we have available screen = update(pyte.Screen(3, 5), @@ -978,6 +1020,7 @@ def test_delete_lines(): [screen.default_char] * 3, [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) # d) with margins -- trying to delete outside scroll boundaries; # expecting nothing to change @@ -996,6 +1039,7 @@ def test_delete_lines(): [Char("b", fg="red"), Char("a", fg="red"), Char("r", fg="red")], [Char("b"), Char("a"), Char("z")], ] + consistency_asserts(screen) def test_insert_characters(): @@ -1052,16 +1096,19 @@ def test_delete_characters(): Char("m", fg="red"), screen.default_char, screen.default_char ] + consistency_asserts(screen) screen.cursor.y, screen.cursor.x = 2, 2 screen.delete_characters() assert (screen.cursor.y, screen.cursor.x) == (2, 2) assert screen.display == ["m ", "is ", "fo "] + consistency_asserts(screen) screen.cursor.y, screen.cursor.x = 1, 1 screen.delete_characters(0) assert (screen.cursor.y, screen.cursor.x) == (1, 1) assert screen.display == ["m ", "i ", "fo "] + consistency_asserts(screen) # ! extreme cases. screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) @@ -1076,6 +1123,7 @@ def test_delete_characters(): screen.default_char, screen.default_char ] + consistency_asserts(screen) screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.cursor.x = 2 @@ -1089,6 +1137,7 @@ def test_delete_characters(): screen.default_char, screen.default_char ] + consistency_asserts(screen) screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.delete_characters(4) @@ -1101,6 +1150,7 @@ def test_delete_characters(): screen.default_char, screen.default_char ] + consistency_asserts(screen) def test_erase_character(): @@ -1114,16 +1164,19 @@ def test_erase_character(): screen.default_char, Char("m", fg="red") ] + consistency_asserts(screen) screen.cursor.y, screen.cursor.x = 2, 2 screen.erase_characters() assert (screen.cursor.y, screen.cursor.x) == (2, 2) assert screen.display == [" m", "is ", "fo "] + consistency_asserts(screen) screen.cursor.y, screen.cursor.x = 1, 1 screen.erase_characters(0) assert (screen.cursor.y, screen.cursor.x) == (1, 1) assert screen.display == [" m", "i ", "fo "] + consistency_asserts(screen) # ! extreme cases. screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) @@ -1138,6 +1191,7 @@ def test_erase_character(): screen.default_char, Char("5", "red") ] + consistency_asserts(screen) screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.cursor.x = 2 @@ -1151,6 +1205,7 @@ def test_erase_character(): screen.default_char, screen.default_char ] + consistency_asserts(screen) screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.erase_characters(4) @@ -1163,6 +1218,7 @@ def test_erase_character(): screen.default_char, Char("5", fg="red") ] + consistency_asserts(screen) def test_erase_in_line(): @@ -1189,6 +1245,7 @@ def test_erase_in_line(): screen.default_char, screen.default_char ] + consistency_asserts(screen) # b) erase from the beginning of the line to the cursor screen = update(screen, @@ -1211,6 +1268,7 @@ def test_erase_in_line(): Char(" ", fg="red"), Char("i", fg="red") ] + consistency_asserts(screen) # c) erase the entire line screen = update(screen, @@ -1227,6 +1285,7 @@ def test_erase_in_line(): "re yo", "u? "] assert tolist(screen)[0] == [screen.default_char] * 5 + consistency_asserts(screen) def test_erase_in_display(): @@ -1256,6 +1315,7 @@ def test_erase_in_display(): [screen.default_char] * 5, [screen.default_char] * 5 ] + consistency_asserts(screen) # b) erase from the beginning of the display to the cursor, # including it @@ -1281,6 +1341,7 @@ def test_erase_in_display(): Char(" ", fg="red"), Char("a", fg="red")], ] + consistency_asserts(screen) # c) erase the while display screen.erase_in_display(2) @@ -1291,6 +1352,7 @@ def test_erase_in_display(): " ", " "] assert tolist(screen) == [[screen.default_char] * 5] * 5 + consistency_asserts(screen) # d) erase with private mode screen = update(pyte.Screen(5, 5), @@ -1305,6 +1367,7 @@ def test_erase_in_display(): " ", " ", " "] + consistency_asserts(screen) # e) erase with extra args screen = update(pyte.Screen(5, 5), @@ -1320,6 +1383,7 @@ def test_erase_in_display(): " ", " ", " "] + consistency_asserts(screen) # f) erase with extra args and private screen = update(pyte.Screen(5, 5), @@ -1334,6 +1398,7 @@ def test_erase_in_display(): " ", " ", " "] + consistency_asserts(screen) def test_cursor_up(): @@ -1462,6 +1527,7 @@ def test_unicode(): stream.feed("тест".encode("utf-8")) assert screen.display == ["тест", " "] + consistency_asserts(screen) def test_alignment_display(): @@ -1477,6 +1543,7 @@ def test_alignment_display(): "b ", " ", " "] + consistency_asserts(screen) screen.alignment_display() @@ -1485,6 +1552,7 @@ def test_alignment_display(): "EEEEE", "EEEEE", "EEEEE"] + consistency_asserts(screen) def test_set_margins(): diff --git a/tests/test_stream.py b/tests/test_stream.py index 7a3ad92..0d05df7 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -1,10 +1,12 @@ -import io +import io, sys, os import pytest import pyte from pyte import charsets as cs, control as ctrl, escape as esc +sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) +from asserts import consistency_asserts class counter: def __init__(self): @@ -227,6 +229,7 @@ def test_define_charset(): stream = pyte.Stream(screen) stream.feed(ctrl.ESC + "(B") assert screen.display[0] == " " * 3 + consistency_asserts(screen) def test_non_utf8_shifts(): @@ -305,6 +308,7 @@ def test_byte_stream_define_charset_unknown(): stream.feed((ctrl.ESC + "(Z").encode()) assert screen.display[0] == " " * 3 assert screen.g0_charset == default_g0_charset + consistency_asserts(screen) @pytest.mark.parametrize("charset,mapping", cs.MAPS.items()) @@ -315,6 +319,7 @@ def test_byte_stream_define_charset(charset, mapping): stream.feed((ctrl.ESC + "(" + charset).encode()) assert screen.display[0] == " " * 3 assert screen.g0_charset == mapping + consistency_asserts(screen) def test_byte_stream_select_other_charset(): From 020fce61c2f5eb5467727048743e481b2a8d2a87 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Mon, 20 Jun 2022 11:16:39 -0300 Subject: [PATCH 10/54] Cache in Char its width Instead of computing it on each screen.display, compute the width of the char once on screen.draw and store it in the Char tuple. This makes screen.display ~x1.10 to ~x1.20 faster and it makes stream.feed only ~x1.01 slower in the worst case. This negative impact is due the change on screen.draw but measurements on my lab show inconsistent results (stream.feed didn't show a consistent performance regression and ~x1.01 slower was the worst value that I've got). --- pyte/screens.py | 22 +++++++++++++--------- tests/helpers/asserts.py | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index cf122ae..796acec 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -71,6 +71,7 @@ class Char(namedtuple("Char", [ "strikethrough", "reverse", "blink", + "width", ])): """A single styled on-screen character. @@ -89,15 +90,16 @@ class Char(namedtuple("Char", [ during rendering. Defaults to ``False``. :param bool blink: flag for rendering the character blinked. Defaults to ``False``. + :param bool width: the width in terms of cells to display this char. """ __slots__ = () - def __new__(cls, data, fg="default", bg="default", bold=False, + def __new__(cls, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, - strikethrough=False, reverse=False, blink=False): + strikethrough=False, reverse=False, blink=False, width=wcwidth(" ")): return super(Char, cls).__new__(cls, data, fg, bg, bold, italics, underscore, strikethrough, reverse, - blink) + blink, width) class Cursor: @@ -111,7 +113,7 @@ class Cursor: """ __slots__ = ("x", "y", "attrs", "hidden") - def __init__(self, x, y, attrs=Char(" ")): + def __init__(self, x, y, attrs=Char(" ", width=wcwidth(" "))): self.x = x self.y = y self.attrs = attrs @@ -211,7 +213,7 @@ class Screen: def default_char(self): """An empty character with default foreground and background colors.""" reverse = mo.DECSCNM in self.mode - return Char(data=" ", fg="default", bg="default", reverse=reverse) + return Char(data=" ", fg="default", bg="default", reverse=reverse, width=wcwidth(" ")) def __init__(self, columns, lines): self.savepoints = [] @@ -256,7 +258,7 @@ def display(self): is_wide_char = False continue char = cell.data - is_wide_char = wcwidth(char[0]) == 2 + is_wide_char = cell.width == 2 display_line.append(char) gap = columns - (prev_x + 1) @@ -527,16 +529,18 @@ def draw(self, data): line = self.buffer[self.cursor.y] if char_width == 1: - line[self.cursor.x] = self.cursor.attrs._replace(data=char) + line[self.cursor.x] = self.cursor.attrs._replace(data=char, width=char_width) elif char_width == 2: # A two-cell character has a stub slot after it. - line[self.cursor.x] = self.cursor.attrs._replace(data=char) + line[self.cursor.x] = self.cursor.attrs._replace(data=char, width=char_width) if self.cursor.x + 1 < self.columns: line[self.cursor.x + 1] = self.cursor.attrs \ - ._replace(data="") + ._replace(data="", width=0) elif char_width == 0 and unicodedata.combining(char): # A zero-cell character is combined with the previous # character either on this or preceding line. + # Because char's width is zero, this will not change the width + # of the previous character. if self.cursor.x: last = line[self.cursor.x - 1] normalized = unicodedata.normalize("NFC", last.data + char) diff --git a/tests/helpers/asserts.py b/tests/helpers/asserts.py index 5ab2689..17aa295 100644 --- a/tests/helpers/asserts.py +++ b/tests/helpers/asserts.py @@ -6,7 +6,17 @@ def consistency_asserts(screen): # width of the first code point. for y in range(screen.lines): for x in range(screen.columns): - char = screen.buffer[y][x].data - assert sum(map(wcwidth, char[1:])) == 0 + data = screen.buffer[y][x].data + assert sum(map(wcwidth, data[1:])) == 0 + # Ensure consistency between the real width (computed here + # with wcwidth(...)) and the char.width attribute + for y in range(screen.lines): + for x in range(screen.columns): + char = screen.buffer[y][x] + if char.data: + assert wcwidth(char.data[0]) == char.width + else: + assert char.data == "" + assert char.width == 0 From 8e7ee070205ef0e740c8b58a6cdca2d53e1a5870 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 25 Jun 2022 23:01:42 -0300 Subject: [PATCH 11/54] Pre-fetch attributes on screen.draw (x1.20 to x2.0 faster) Fetch some attributes that were frequently accessed in the for-loop of screen.draw avoiding accessing them on each iteration. Most of them remain constant within the draw() method anyways. Others, like cursor.x, cursor.y and line are updated infrequently inside the for-loop so it still faster pre-fetch them outside and update them if needed than accessing them on each iteration. Benchmark results show stream.feed is x1.20 to x2.0 faster with these optimizations. Benchmark files that have more control sequences (like htop, mc and vim) have a lower improvement as the parsing of these sequences dominates the runtime of stream.feed. --- pyte/screens.py | 68 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 796acec..657821f 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -506,6 +506,28 @@ def draw(self, data): data = data.translate( self.g1_charset if self.charset else self.g0_charset) + # Fetch these attributes to avoid a lookup on each iteration + # of the for-loop. + # These attributes are expected to be constant across all the + # execution of self.draw() + columns = self.columns + cursor = self.cursor + buffer = self.buffer + attrs = cursor.attrs + mode = self.mode + + # Note: checking for IRM here makes sense because it would be + # checked on every char in data otherwise. + # Checking DECAWM, on the other hand, not necessary is a good + # idea because it only matters if cursor_x == columns (unlikely) + is_IRM_set = mo.IRM in mode + DECAWM = mo.DECAWM + + # The following are attributes expected to change infrequently + # so we fetch them here and update accordingly if necessary + cursor_x = cursor.x + cursor_y = cursor.y + line = buffer[cursor_y] for char in data: char_width = wcwidth(char) @@ -513,52 +535,62 @@ def draw(self, data): # enabled, move the cursor to the beginning of the next line, # otherwise replace characters already displayed with newly # entered. - if self.cursor.x == self.columns: - if mo.DECAWM in self.mode: - self.dirty.add(self.cursor.y) + if cursor_x == columns: + if DECAWM in mode: + self.dirty.add(cursor_y) self.carriage_return() self.linefeed() + + # carriage_return implies cursor.x = 0 so we update cursor_x + cursor_x = 0 + + # linefeed may update cursor_y so we update cursor_y too + cursor_y = cursor.y + line = buffer[cursor_y] elif char_width > 0: - self.cursor.x -= char_width + cursor_x -= char_width + cursor.x = cursor_x # If Insert mode is set, new characters move old characters to # the right, otherwise terminal is in Replace mode and new # characters replace old characters at cursor position. - if mo.IRM in self.mode and char_width > 0: + if is_IRM_set and char_width > 0: self.insert_characters(char_width) - line = self.buffer[self.cursor.y] if char_width == 1: - line[self.cursor.x] = self.cursor.attrs._replace(data=char, width=char_width) + line[cursor_x] = attrs._replace(data=char, width=char_width) elif char_width == 2: # A two-cell character has a stub slot after it. - line[self.cursor.x] = self.cursor.attrs._replace(data=char, width=char_width) - if self.cursor.x + 1 < self.columns: - line[self.cursor.x + 1] = self.cursor.attrs \ + line[cursor_x] = attrs._replace(data=char, width=char_width) + if cursor_x + 1 < columns: + line[cursor_x + 1] = attrs \ ._replace(data="", width=0) elif char_width == 0 and unicodedata.combining(char): # A zero-cell character is combined with the previous # character either on this or preceding line. # Because char's width is zero, this will not change the width # of the previous character. - if self.cursor.x: - last = line[self.cursor.x - 1] + if cursor_x: + last = line[cursor_x - 1] normalized = unicodedata.normalize("NFC", last.data + char) - line[self.cursor.x - 1] = last._replace(data=normalized) - elif self.cursor.y: - last = self.buffer[self.cursor.y - 1][self.columns - 1] + line[cursor_x - 1] = last._replace(data=normalized) + elif cursor_y: + last = buffer[cursor_y - 1][columns - 1] normalized = unicodedata.normalize("NFC", last.data + char) - self.buffer[self.cursor.y - 1][self.columns - 1] = \ + buffer[cursor_y - 1][columns - 1] = \ last._replace(data=normalized) else: + cursor.x = cursor_x + cursor.y = cursor_y break # Unprintable character or doesn't advance the cursor. # .. note:: We can't use :meth:`cursor_forward()`, because that # way, we'll never know when to linefeed. if char_width > 0: - self.cursor.x = min(self.cursor.x + char_width, self.columns) + cursor_x = min(cursor_x + char_width, columns) + cursor.x = cursor_x - self.dirty.add(self.cursor.y) + self.dirty.add(cursor_y) def set_title(self, param): """Set terminal title. From d4d2e4a5e52f1724aca8cff69e5e48cfc3c84647 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 11:26:41 -0300 Subject: [PATCH 12/54] Allow temporal cursor_x > columns (x1.05 to x1.14 faster) Instead of checking if cursor_x > columns at the end of iteration and set cursor_x to the minimum of (cursor_x and columns), delay that decision to the begin of the next iteration or at the end of the for-loop. This removes one "if" statement at the end of the for-loop and allows us to use the local variable cursor_x all the time without having to update cursor.x. Only this happens before insert_characters() and at the end of the draw() method when the cursor.x is been visible by code outside draw() and therefore must be updated with the latest value of cursor_x. This optimization makes stream.feed between x1.05 and x1.14 faster. As in any optimization on draw(), the use cases that gets more improvements are the ones that have very few control sequences in their input (so stream.feed is dominated by screen.draw and not be stream._parse_fsm) --- pyte/screens.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 657821f..c2e689a 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -535,26 +535,42 @@ def draw(self, data): # enabled, move the cursor to the beginning of the next line, # otherwise replace characters already displayed with newly # entered. - if cursor_x == columns: + if cursor_x >= columns: if DECAWM in mode: self.dirty.add(cursor_y) self.carriage_return() self.linefeed() # carriage_return implies cursor.x = 0 so we update cursor_x + # This also puts the cursor_x back into the screen if before + # cursor_x was outside (cursor_x > columns). See the comments + # at the end of the for-loop cursor_x = 0 - # linefeed may update cursor_y so we update cursor_y too + # linefeed may update cursor.y so we update cursor_y and + # the current line accordingly. cursor_y = cursor.y line = buffer[cursor_y] elif char_width > 0: - cursor_x -= char_width - cursor.x = cursor_x + # Move the cursor_x back enough to make room for + # the new char. + # This indirectly fixes the case of cursor_x > columns putting + # the cursor_x back to the screen. + cursor_x = columns - char_width + else: + # Ensure that cursor_x = min(cursor_x, columns) in the case + # that wcwidth returned 0 or negative and the flow didn't enter + # in any of the branches above. + # See the comments at the end of the for-loop + cursor_x = columns # If Insert mode is set, new characters move old characters to # the right, otherwise terminal is in Replace mode and new # characters replace old characters at cursor position. if is_IRM_set and char_width > 0: + # update the real cursor so insert_characters() can use + # an updated (and correct) value of it + cursor.x = cursor_x self.insert_characters(char_width) if char_width == 1: @@ -580,18 +596,26 @@ def draw(self, data): buffer[cursor_y - 1][columns - 1] = \ last._replace(data=normalized) else: - cursor.x = cursor_x - cursor.y = cursor_y break # Unprintable character or doesn't advance the cursor. # .. note:: We can't use :meth:`cursor_forward()`, because that # way, we'll never know when to linefeed. - if char_width > 0: - cursor_x = min(cursor_x + char_width, columns) - cursor.x = cursor_x + # + # Note: cursor_x may leave outside the screen if cursor_x > columns + # but this is going to be fixed in the next iteration or at the end + # of the draw() method + cursor_x += char_width self.dirty.add(cursor_y) + # Update the real cursor fixing the cursor_x to be + # within the limits of the screen + if cursor_x > columns: + cursor.x = columns + else: + cursor.x = cursor_x + cursor.y = cursor_y + def set_title(self, param): """Set terminal title. From 945b19bbf9555b565ea199157c405bd603915ca0 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 14:51:48 -0300 Subject: [PATCH 13/54] Make Char mutable (not a namedtuple) (API may break) Make Char mutable (and ordinary object) so we can modify each char in place avoiding calling _replace. This commit only changed the Char class and implements some methods to emulate the namedtuple API. Theoretically it could be possible to emulate the whole namedtuple API but it is unclear if it worth. In this scenario, user code may break. Using a plain object instead of a namedtuple added a regression on memory usage of x1.20 for htop and mc benchmark files when HistoryScreen was used. The rest of the benchmarks didn't change significantly (but it is expected to be slightly more inefficient). --- pyte/screens.py | 64 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index c2e689a..0cdb1cf 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -61,18 +61,7 @@ ]) -class Char(namedtuple("Char", [ - "data", - "fg", - "bg", - "bold", - "italics", - "underscore", - "strikethrough", - "reverse", - "blink", - "width", -])): +class Char: """A single styled on-screen character. :param str data: unicode character. Invariant: ``len(data) == 1``. @@ -92,15 +81,54 @@ class Char(namedtuple("Char", [ ``False``. :param bool width: the width in terms of cells to display this char. """ - __slots__ = () + __slots__ = ( + "data", + "fg", + "bg", + "bold", + "italics", + "underscore", + "strikethrough", + "reverse", + "blink", + "width", + ) - def __new__(cls, data=" ", fg="default", bg="default", bold=False, + def __init__(self, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, strikethrough=False, reverse=False, blink=False, width=wcwidth(" ")): - return super(Char, cls).__new__(cls, data, fg, bg, bold, italics, - underscore, strikethrough, reverse, - blink, width) - + self.data = data + self.fg = fg + self.bg = bg + self.bold = bold + self.italics = italics + self.underscore = underscore + self.strikethrough = strikethrough + self.reverse = reverse + self.blink = blink + self.width = width + + def _replace(self, **kargs): + fields = self._asdict() + fields.update(kargs) + return Char(**fields) + + def _asdict(self): + return {name: getattr(self, name) for name in self.__slots__} + + def __eq__(self, other): + if not isinstance(other, Char): + raise TypeError() + + return all(getattr(self, name) == getattr(other, name) for name in self.__slots__) + + def __ne__(self, other): + if not isinstance(other, Char): + raise TypeError() + + return any(getattr(self, name) != getattr(other, name) for name in self.__slots__) + + _fields = __slots__ class Cursor: """Screen cursor. From e881d2542d252e99cc6477b00e20c2b6aef4fc7b Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 17:39:46 -0300 Subject: [PATCH 14/54] Refactor Char's style in a separated namedtuple object. --- pyte/screens.py | 73 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 0cdb1cf..c9625fd 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -60,6 +60,17 @@ "wrap" ]) +CharStyle = namedtuple("CharStyle", [ + "fg", + "bg", + "bold", + "italics", + "underscore", + "strikethrough", + "reverse", + "blink", +]) + class Char: """A single styled on-screen character. @@ -83,30 +94,52 @@ class Char: """ __slots__ = ( "data", - "fg", - "bg", - "bold", - "italics", - "underscore", - "strikethrough", - "reverse", - "blink", "width", + "style", ) + # List the properties of this Char instance including its style's properties + # The order of this _fields is maintained for backward compatibility + _fields = ("data",) + CharStyle._fields + ("width",) + def __init__(self, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, strikethrough=False, reverse=False, blink=False, width=wcwidth(" ")): self.data = data - self.fg = fg - self.bg = bg - self.bold = bold - self.italics = italics - self.underscore = underscore - self.strikethrough = strikethrough - self.reverse = reverse - self.blink = blink self.width = width + self.style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) + + @property + def fg(self): + return self.style.fg + + @property + def bg(self): + return self.style.bg + + @property + def bold(self): + return self.style.bold + + @property + def italics(self): + return self.style.italics + + @property + def underscore(self): + return self.style.underscore + + @property + def strikethrough(self): + return self.style.strikethrough + + @property + def reverse(self): + return self.style.reverse + + @property + def blink(self): + return self.style.blink def _replace(self, **kargs): fields = self._asdict() @@ -114,21 +147,19 @@ def _replace(self, **kargs): return Char(**fields) def _asdict(self): - return {name: getattr(self, name) for name in self.__slots__} + return {name: getattr(self, name) for name in self._fields} def __eq__(self, other): if not isinstance(other, Char): raise TypeError() - return all(getattr(self, name) == getattr(other, name) for name in self.__slots__) + return all(getattr(self, name) == getattr(other, name) for name in self._fields) def __ne__(self, other): if not isinstance(other, Char): raise TypeError() - return any(getattr(self, name) != getattr(other, name) for name in self.__slots__) - - _fields = __slots__ + return any(getattr(self, name) != getattr(other, name) for name in self._fields) class Cursor: """Screen cursor. From 5f784ec4672e5720f436774ead9afb32a8d1f4cb Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 18:29:35 -0300 Subject: [PATCH 15/54] Reuse/share char styles (x1.05 to x1.30 lighter) Reduce the memory footprint reusing/sharing the same CharStyle object among different Char instances. A specialized _replace_data changes the data and width of the char but not its style. This reduces the footprint between x1.05 and x1.30 with respect the 0.8.1-memory.json baseline results. --- pyte/screens.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index c9625fd..1360fbf 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -104,10 +104,13 @@ class Char: def __init__(self, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, - strikethrough=False, reverse=False, blink=False, width=wcwidth(" ")): + strikethrough=False, reverse=False, blink=False, width=wcwidth(" "), style=None): self.data = data self.width = width - self.style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) + if style: + self.style = style + else: + self.style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) @property def fg(self): @@ -146,6 +149,9 @@ def _replace(self, **kargs): fields.update(kargs) return Char(**fields) + def _replace_data(self, data, width): + return Char(data=data, width=width, style=self.style) + def _asdict(self): return {name: getattr(self, name) for name in self._fields} @@ -271,8 +277,9 @@ class Screen: @property def default_char(self): """An empty character with default foreground and background colors.""" - reverse = mo.DECSCNM in self.mode - return Char(data=" ", fg="default", bg="default", reverse=reverse, width=wcwidth(" ")) + style = self._default_style_reversed if mo.DECSCNM in self.mode else self._default_style + return Char(data=" ", width=wcwidth(" "), style=style) + def __init__(self, columns, lines): self.savepoints = [] @@ -282,6 +289,12 @@ def __init__(self, columns, lines): self.dirty = set() self.reset() + self._default_style = CharStyle( + fg="default", bg="default", bold=False, + italics=False, underscore=False, + strikethrough=False, reverse=False, blink=False) + self._default_style_reversed = self._default_style._replace(reverse=True) + def __repr__(self): return ("{0}({1}, {2})".format(self.__class__.__name__, self.columns, self.lines)) @@ -633,13 +646,13 @@ def draw(self, data): self.insert_characters(char_width) if char_width == 1: - line[cursor_x] = attrs._replace(data=char, width=char_width) + line[cursor_x] = attrs._replace_data(data=char, width=char_width) elif char_width == 2: # A two-cell character has a stub slot after it. - line[cursor_x] = attrs._replace(data=char, width=char_width) + line[cursor_x] = attrs._replace_data(data=char, width=char_width) if cursor_x + 1 < columns: line[cursor_x + 1] = attrs \ - ._replace(data="", width=0) + ._replace_data(data="", width=0) elif char_width == 0 and unicodedata.combining(char): # A zero-cell character is combined with the previous # character either on this or preceding line. @@ -648,12 +661,12 @@ def draw(self, data): if cursor_x: last = line[cursor_x - 1] normalized = unicodedata.normalize("NFC", last.data + char) - line[cursor_x - 1] = last._replace(data=normalized) + line[cursor_x - 1] = last._replace_data(data=normalized, width=last.width) elif cursor_y: last = buffer[cursor_y - 1][columns - 1] normalized = unicodedata.normalize("NFC", last.data + char) buffer[cursor_y - 1][columns - 1] = \ - last._replace(data=normalized) + last._replace_data(data=normalized, width=last.width) else: break # Unprintable character or doesn't advance the cursor. @@ -1109,7 +1122,7 @@ def alignment_display(self): self.dirty.update(range(self.lines)) for y in range(self.lines): for x in range(self.columns): - self.buffer[y][x] = self.buffer[y][x]._replace(data="E") + self.buffer[y][x] = self.buffer[y][x]._replace_data(data="E", width=wcwidth("E")) def select_graphic_rendition(self, *attrs): """Set display attributes. From 9721698a3132f67ee78c976054d847237be973ca Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 26 Jun 2022 21:50:52 -0300 Subject: [PATCH 16/54] Update Chars in-place instead of recreating them (x1.20 and x1.90 faster; regress on mem) Instead of calling _replace() to create a new Char object, modify the existing one. For that, the Line (ex StaticDefaultDict) is in charge to fetch the char and do the modifications. If no Char is found, only then a Char is created and inserted in the Line (dict). See write_data(). In some cases we need to get a Char, read it and then update it so a copy of the Line's default char is returned and added to the line. A copy is required because now the Char are mutable. See char_at(). Changed the API of Char: _asdict renamed as as_dict and _replace as copy_and_change; removed _replace_data and added a copy method. The constructor also changed: it is required data, width and style. The former way to construct a Char can be done with from_attributes class method. This commits improved the runtime of stream.feed by x1.20 to x1.90 (faster) however a regression on the memory footprint was found (between x1.10 and x1.50). I don't have an explanation for this last point. --- pyte/screens.py | 124 +++++++++++++++++++++++++++++-------------- tests/test_screen.py | 20 +++++-- 2 files changed, 99 insertions(+), 45 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 1360fbf..c3a6b15 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -102,15 +102,16 @@ class Char: # The order of this _fields is maintained for backward compatibility _fields = ("data",) + CharStyle._fields + ("width",) - def __init__(self, data=" ", fg="default", bg="default", bold=False, - italics=False, underscore=False, - strikethrough=False, reverse=False, blink=False, width=wcwidth(" "), style=None): + def __init__(self, data, width, style): self.data = data self.width = width - if style: - self.style = style - else: - self.style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) + self.style = style + + @classmethod + def from_attributes(cls, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, + strikethrough=False, reverse=False, blink=False): + style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) + return Char(data, wcwidth(data), style) @property def fg(self): @@ -144,15 +145,15 @@ def reverse(self): def blink(self): return self.style.blink - def _replace(self, **kargs): + def copy_and_change(self, **kargs): fields = self._asdict() fields.update(kargs) return Char(**fields) - def _replace_data(self, data, width): - return Char(data=data, width=width, style=self.style) + def copy(self): + return Char(self.data, self.width, self.style) - def _asdict(self): + def as_dict(self): return {name: getattr(self, name) for name in self._fields} def __eq__(self, other): @@ -178,20 +179,20 @@ class Cursor: """ __slots__ = ("x", "y", "attrs", "hidden") - def __init__(self, x, y, attrs=Char(" ", width=wcwidth(" "))): + def __init__(self, x, y, attrs): self.x = x self.y = y self.attrs = attrs self.hidden = False -class StaticDefaultDict(dict): - """A :func:`dict` with a static default value. +class Line(dict): + """A :func:`dict` with a static default value representing a line of the screen. Unlike :func:`collections.defaultdict` this implementation does not implicitly update the mapping when queried with a missing key. - >>> d = StaticDefaultDict(42) + >>> d = Line(42) >>> d["foo"] 42 >>> d @@ -203,6 +204,28 @@ def __init__(self, default): def __missing__(self, key): return self.default + def write_data(self, x, data, width, style): + """ + Update the char at the position x with the new data, width and style. + If no char is at that position, a new char is created. + """ + if x in self: + char = self[x] + char.data = data + char.width = width + char.style = style + else: + self[x] = Char(data, width, style) + + def char_at(self, x): + if x in self: + return self[x] + else: + char = self.default.copy() + self[x] = char + return char + + class Screen: """ @@ -278,16 +301,15 @@ class Screen: def default_char(self): """An empty character with default foreground and background colors.""" style = self._default_style_reversed if mo.DECSCNM in self.mode else self._default_style - return Char(data=" ", width=wcwidth(" "), style=style) + return Char(" ", wcwidth(" "), style) def __init__(self, columns, lines): self.savepoints = [] self.columns = columns self.lines = lines - self.buffer = defaultdict(lambda: StaticDefaultDict(self.default_char)) + self.buffer = defaultdict(lambda: Line(self.default_char)) self.dirty = set() - self.reset() self._default_style = CharStyle( fg="default", bg="default", bold=False, @@ -295,6 +317,8 @@ def __init__(self, columns, lines): strikethrough=False, reverse=False, blink=False) self._default_style_reversed = self._default_style._replace(reverse=True) + self.reset() + def __repr__(self): return ("{0}({1}, {2})".format(self.__class__.__name__, self.columns, self.lines)) @@ -380,7 +404,7 @@ def reset(self): # we aim to support VT102 / VT220 and linux -- we use n = 8. self.tabstops = set(range(8, self.columns, 8)) - self.cursor = Cursor(0, 0) + self.cursor = Cursor(0, 0, self.default_char.copy()) self.cursor_position() self.saved_columns = None @@ -493,7 +517,7 @@ def set_mode(self, *modes, **kwargs): for line in self.buffer.values(): line.default = self.default_char for x in line: - line[x] = line[x]._replace(reverse=True) + line[x].style = line[x].style._replace(reverse=True) self.select_graphic_rendition(7) # +reverse. @@ -531,7 +555,7 @@ def reset_mode(self, *modes, **kwargs): for line in self.buffer.values(): line.default = self.default_char for x in line: - line[x] = line[x]._replace(reverse=False) + line[x].style = line[x].style._replace(reverse=False) self.select_graphic_rendition(27) # -reverse. @@ -587,6 +611,7 @@ def draw(self, data): buffer = self.buffer attrs = cursor.attrs mode = self.mode + style = attrs.style # Note: checking for IRM here makes sense because it would be # checked on every char in data otherwise. @@ -646,27 +671,25 @@ def draw(self, data): self.insert_characters(char_width) if char_width == 1: - line[cursor_x] = attrs._replace_data(data=char, width=char_width) + line.write_data(cursor_x, char, char_width, style) elif char_width == 2: # A two-cell character has a stub slot after it. - line[cursor_x] = attrs._replace_data(data=char, width=char_width) + line.write_data(cursor_x, char, char_width, style) if cursor_x + 1 < columns: - line[cursor_x + 1] = attrs \ - ._replace_data(data="", width=0) + line.write_data(cursor_x+1, "", 0, style) elif char_width == 0 and unicodedata.combining(char): # A zero-cell character is combined with the previous # character either on this or preceding line. # Because char's width is zero, this will not change the width # of the previous character. if cursor_x: - last = line[cursor_x - 1] + last = line.char_at(cursor_x - 1) normalized = unicodedata.normalize("NFC", last.data + char) - line[cursor_x - 1] = last._replace_data(data=normalized, width=last.width) + last.data = normalized elif cursor_y: - last = buffer[cursor_y - 1][columns - 1] + last = buffer[cursor_y - 1].char_at(columns - 1) normalized = unicodedata.normalize("NFC", last.data + char) - buffer[cursor_y - 1][columns - 1] = \ - last._replace_data(data=normalized, width=last.width) + last.data = normalized else: break # Unprintable character or doesn't advance the cursor. @@ -764,7 +787,7 @@ def backspace(self): def save_cursor(self): """Push the current cursor position onto the stack.""" - self.savepoints.append(Savepoint(copy.copy(self.cursor), + self.savepoints.append(Savepoint(copy.deepcopy(self.cursor), self.g0_charset, self.g1_charset, self.charset, @@ -852,9 +875,18 @@ def insert_characters(self, count=None): count = count or 1 line = self.buffer[self.cursor.y] for x in range(self.columns, self.cursor.x - 1, -1): - if x + count <= self.columns: - line[x + count] = line[x] - line.pop(x, None) + new_x = x + count + if new_x <= self.columns: + if x in line: + line[x + count] = line.pop(x) + else: + # this is equivalent to: + # line[new_x] = line[x] + # where line[x] does not exist so line[new_x] + # should not exist either + line.pop(new_x, None) + else: + line.pop(x, None) def delete_characters(self, count=None): """Delete the indicated # of characters, starting with the @@ -870,7 +902,7 @@ def delete_characters(self, count=None): line = self.buffer[self.cursor.y] for x in range(self.cursor.x, self.columns): if x + count <= self.columns: - line[x] = line.pop(x + count, self.default_char) + line[x] = line.pop(x + count, self.default_char.copy()) else: line.pop(x, None) @@ -892,9 +924,12 @@ def erase_characters(self, count=None): count = count or 1 line = self.buffer[self.cursor.y] + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style for x in range(self.cursor.x, min(self.cursor.x + count, self.columns)): - line[x] = self.cursor.attrs + line.write_data(x, data, width, style) def erase_in_line(self, how=0, private=False): """Erase a line in a specific way. @@ -920,8 +955,11 @@ def erase_in_line(self, how=0, private=False): interval = range(self.columns) line = self.buffer[self.cursor.y] + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style for x in interval: - line[x] = self.cursor.attrs + line.write_data(x, data, width, style) def erase_in_display(self, how=0, *args, **kwargs): """Erases display in a specific way. @@ -954,10 +992,13 @@ def erase_in_display(self, how=0, *args, **kwargs): interval = range(self.lines) self.dirty.update(interval) + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style for y in interval: line = self.buffer[y] for x in line: - line[x] = self.cursor.attrs + line.write_data(x, data, width, style) if how == 0 or how == 1: self.erase_in_line(how) @@ -1120,9 +1161,10 @@ def bell(self, *args): def alignment_display(self): """Fills screen with uppercase E's for screen focus and alignment.""" self.dirty.update(range(self.lines)) + style = self._default_style for y in range(self.lines): for x in range(self.columns): - self.buffer[y][x] = self.buffer[y][x]._replace_data(data="E", width=wcwidth("E")) + self.buffer[y].write_data(x, "E", wcwidth("E"), style) def select_graphic_rendition(self, *attrs): """Set display attributes. @@ -1142,7 +1184,7 @@ def select_graphic_rendition(self, *attrs): attr = attrs.pop() if attr == 0: # Reset all attributes. - replace.update(self.default_char._asdict()) + replace.update(self.default_char.style._asdict()) elif attr in g.FG_ANSI: replace["fg"] = g.FG_ANSI[attr] elif attr in g.BG: @@ -1170,7 +1212,7 @@ def select_graphic_rendition(self, *attrs): except IndexError: pass - self.cursor.attrs = self.cursor.attrs._replace(**replace) + self.cursor.attrs.style = self.cursor.attrs.style._replace(**replace) def report_device_attributes(self, mode=0, **kwargs): """Report terminal identity. diff --git a/tests/test_screen.py b/tests/test_screen.py index b4fe75a..eb820f8 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -4,24 +4,36 @@ import pyte from pyte import modes as mo, control as ctrl, graphics as g -from pyte.screens import Char +from pyte.screens import Char as _orig_Char, CharStyle sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) from asserts import consistency_asserts +# Implement the old API of Char so we don't have to change +# all the tests +class Char(_orig_Char): + def __init__(self, data=" ", fg="default", bg="default", bold=False, italics=False, underscore=False, + strikethrough=False, reverse=False, blink=False, width=1): + self.data = data + self.width = width + self.style = CharStyle(fg, bg, bold, italics, underscore, strikethrough, reverse, blink) + + # Test helpers. def update(screen, lines, colored=[]): """Updates a given screen object with given lines, colors each line from ``colored`` in "red" and returns the modified screen. """ + base_style = Char().style + red_style = base_style._replace(fg="red") for y, line in enumerate(lines): for x, char in enumerate(line): if y in colored: - attrs = {"fg": "red"} + style = red_style else: - attrs = {} - screen.buffer[y][x] = Char(data=char, **attrs) + style = base_style + screen.buffer[y].write_data(x, char, 1, style) return screen From e49fb3f840852af24a72c500104121eabe380cf7 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Tue, 28 Jun 2022 22:43:38 -0300 Subject: [PATCH 17/54] Fix test_reverse_index (history) due old API The test was using a legacy API of screen.buffer when the buffer was a dense matrix. Now it is sparse we cannot use len(screen.buffer) anymore or buffer[-1] either. --- tests/test_history.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_history.py b/tests/test_history.py index 288b940..00e668a 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -48,9 +48,9 @@ def test_reverse_index(): # Filling the screen with line numbers, so it's easier to # track history contents. - for idx in range(len(screen.buffer)): + for idx in range(screen.lines): screen.draw(str(idx)) - if idx != len(screen.buffer) - 1: + if idx != screen.lines - 1: screen.linefeed() assert not screen.history.top @@ -59,19 +59,19 @@ def test_reverse_index(): screen.cursor_position() # a) first index, expecting top history to be updated. - line = screen.buffer[-1] + line = screen.buffer[screen.lines-1] screen.reverse_index() assert screen.history.bottom assert screen.history.bottom[0] == line # b) second index. - line = screen.buffer[-1] + line = screen.buffer[screen.lines-1] screen.reverse_index() assert len(screen.history.bottom) == 2 assert screen.history.bottom[1] == line # c) rotation. - for _ in range(len(screen.buffer) ** screen.lines): + for _ in range(screen.history.size * 2): screen.reverse_index() assert len(screen.history.bottom) == 50 From d94299df031fc5ab9dfbfef09586f4988a24b166 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Tue, 28 Jun 2022 23:22:25 -0300 Subject: [PATCH 18/54] Use binary search over non-empty lines on index/reverse_index This improvement impacts slighly negatively over small geometries (x1.01 to x1.05 slower) but improves on larger geometries and for almost all the cases of HistoryScreen (x1.10 to x1.20) --- pyte/screens.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index c3a6b15..b2543ce 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -34,6 +34,7 @@ import warnings from collections import deque, namedtuple, defaultdict from functools import lru_cache +from bisect import bisect_left, bisect_right from wcwidth import wcwidth @@ -735,11 +736,20 @@ def index(self): """ top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == bottom: + buffer = self.buffer + pop = buffer.pop + + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, top + 1) + end = bisect_right(non_empty_y, bottom, begin) + + to_move = non_empty_y[begin:end] + for y in to_move: + buffer[y-1] = pop(y) + # TODO: mark only the lines within margins? + # we could mark "(y-1, y) for y in to_move" self.dirty.update(range(self.lines)) - for y in range(top, bottom): - self.buffer[y] = self.buffer[y + 1] - self.buffer.pop(bottom, None) else: self.cursor_down() @@ -749,11 +759,21 @@ def reverse_index(self): """ top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == top: + buffer = self.buffer + pop = buffer.pop + + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, top) + end = bisect_right(non_empty_y, bottom - 1, begin) + + to_move = non_empty_y[begin:end] + for y in reversed(to_move): + buffer[y+1] = pop(y) + # TODO: mark only the lines within margins? + # we could mark "(y+1, y) for y in to_move" self.dirty.update(range(self.lines)) - for y in range(bottom, top, -1): - self.buffer[y] = self.buffer[y - 1] - self.buffer.pop(top, None) + else: self.cursor_up() From 912028fdf8ac8b6399bea6dd4e727808044033ec Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 1 Jul 2022 23:38:12 -0300 Subject: [PATCH 19/54] Minor optimizations. --- pyte/screens.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index b2543ce..76d9188 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -199,6 +199,7 @@ class Line(dict): >>> d {} """ + __slots__ = ('default', ) def __init__(self, default): self.default = default @@ -227,7 +228,6 @@ def char_at(self, x): return char - class Screen: """ A screen is an in-memory matrix of characters that represents the @@ -626,6 +626,8 @@ def draw(self, data): cursor_x = cursor.x cursor_y = cursor.y line = buffer[cursor_y] + write_data = line.write_data + char_at = line.char_at for char in data: char_width = wcwidth(char) @@ -649,6 +651,8 @@ def draw(self, data): # the current line accordingly. cursor_y = cursor.y line = buffer[cursor_y] + write_data = line.write_data + char_at = line.char_at elif char_width > 0: # Move the cursor_x back enough to make room for # the new char. @@ -672,19 +676,19 @@ def draw(self, data): self.insert_characters(char_width) if char_width == 1: - line.write_data(cursor_x, char, char_width, style) + write_data(cursor_x, char, char_width, style) elif char_width == 2: # A two-cell character has a stub slot after it. - line.write_data(cursor_x, char, char_width, style) + write_data(cursor_x, char, char_width, style) if cursor_x + 1 < columns: - line.write_data(cursor_x+1, "", 0, style) + write_data(cursor_x+1, "", 0, style) elif char_width == 0 and unicodedata.combining(char): # A zero-cell character is combined with the previous # character either on this or preceding line. # Because char's width is zero, this will not change the width # of the previous character. if cursor_x: - last = line.char_at(cursor_x - 1) + last = char_at(cursor_x - 1) normalized = unicodedata.normalize("NFC", last.data + char) last.data = normalized elif cursor_y: @@ -943,13 +947,13 @@ def erase_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - line = self.buffer[self.cursor.y] + write_data = self.buffer[self.cursor.y].write_data data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style for x in range(self.cursor.x, min(self.cursor.x + count, self.columns)): - line.write_data(x, data, width, style) + write_data(x, data, width, style) def erase_in_line(self, how=0, private=False): """Erase a line in a specific way. @@ -974,12 +978,12 @@ def erase_in_line(self, how=0, private=False): elif how == 2: interval = range(self.columns) - line = self.buffer[self.cursor.y] + write_data = self.buffer[self.cursor.y].write_data data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style for x in interval: - line.write_data(x, data, width, style) + write_data(x, data, width, style) def erase_in_display(self, how=0, *args, **kwargs): """Erases display in a specific way. @@ -1017,8 +1021,9 @@ def erase_in_display(self, how=0, *args, **kwargs): style = self.cursor.attrs.style for y in interval: line = self.buffer[y] + write_data = line.write_data for x in line: - line.write_data(x, data, width, style) + write_data(x, data, width, style) if how == 0 or how == 1: self.erase_in_line(how) @@ -1394,9 +1399,10 @@ def after_event(self, event): """ if event in ["prev_page", "next_page"]: for line in self.buffer.values(): + pop = line.pop for x in line: if x > self.columns: - line.pop(x) + pop(x) # If we're at the bottom of the history buffer and `DECTCEM` # mode is set -- show the cursor. From 4c04935c735a7484c90fce4a166aa29686998f12 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 2 Jul 2022 19:10:07 -0300 Subject: [PATCH 20/54] Calculate statistics about buffer's and lines' internals (no stable API) It is handy to get some stats about the layout and chars locations in the lines/buffer and see how sparse they are. The statistics are not part of the stable API so they may change between versions. --- pyte/screens.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pyte/screens.py b/pyte/screens.py index 76d9188..54f54c3 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -72,6 +72,71 @@ "blink", ]) +class LineStats(namedtuple("_LineStats", [ + "empty", + "chars", + "columns", + "occupacy", + "min", + "max", + "span", + ])): + """ + Note: this is not part of the stable API so it may change + between version of pyte. + """ + + def __repr__(self): + if self.empty: + return "chars: {0: >3}/{1} ({2:.2f})".format( + self.chars, self.columns, self.occupacy, + ) + else: + return "chars: {0: >3}/{1} ({2:.2f}); range: [{3: >3} - {4: >3}], len: {5: >3} ({6:.2f})".format( + self.chars, self.columns, self.occupacy, + self.min, self.max, self.span, self.span/self.columns + ) + +class BufferStats(namedtuple("_BufferStats", [ + "empty", + "entries", + "columns", + "lines", + "falses", + "occupacy", + "min", + "max", + "span", + "line_stats", + ])): + """ + Note: this is not part of the stable API so it may change + between version of pyte. + """ + + def __repr__(self): + total_chars = sum(stats.chars for _, stats in self.line_stats) + bstats = "total chars: {0: >3}/{1} ({2:.2f}%)\n".format( + total_chars, self.columns*self.lines, + total_chars/(self.columns*self.lines) + ) + + if self.empty: + return bstats + \ + "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f})\n{5}".format( + self.entries, self.lines, self.occupacy, + self.falses, self.falses/self.entries, + "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) + ) + else: + return bstats + \ + "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}); range: [{5: >3} - {6: >3}], len: {7: >3} ({8:.2f})\n{9}".format( + self.entries, self.lines, self.occupacy, + self.falses, self.falses/self.entries, + self.min, self.max, self.span, self.span/self.lines, + "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) + ) + class Char: """A single styled on-screen character. @@ -227,6 +292,20 @@ def char_at(self, x): self[x] = char return char + def stats(self, screen): + """ + Note: this is not part of the stable API so it may change + between version of pyte. + """ + return LineStats( + empty=not bool(self), + chars=len(self), + columns=screen.columns, + occupacy=len(self)/screen.columns, + min=min(self) if self else None, + max=max(self) if self else None, + span=(max(self) - min(self)) if self else None + ) class Screen: """ @@ -323,6 +402,24 @@ def __init__(self, columns, lines): def __repr__(self): return ("{0}({1}, {2})".format(self.__class__.__name__, self.columns, self.lines)) + def stats(self): + """ + Note: this is not part of the stable API so it may change + between version of pyte. + """ + buffer = self.buffer + return BufferStats( + empty=not bool(buffer), + entries=len(buffer), + columns=self.columns, + lines=self.lines, + falses=len([line for line in buffer.values() if not line]), + occupacy=len(buffer)/self.lines, + min=min(buffer) if buffer else None, + max=max(buffer) if buffer else None, + span=(max(buffer) - min(buffer)) if buffer else None, + line_stats=[(x, line.stats(self)) for x, line in sorted(buffer.items())] + ) @property def display(self): From 84cd21f61f3948aaf2b44181d9df6de5811aebe3 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Mon, 4 Jul 2022 19:49:10 -0300 Subject: [PATCH 21/54] On screen.buffer return a read-only view (BufferView/LineView) This layer of abstraction will allow use to changes on the real buffer without breaking the public API. --- pyte/screens.py | 115 ++++++++++++++++++++++++++++++------------ tests/test_history.py | 8 +-- tests/test_screen.py | 4 +- 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 54f54c3..6ecb982 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -307,6 +307,54 @@ def stats(self, screen): span=(max(self) - min(self)) if self else None ) + +class LineView: + """ + A read-only view of an horizontal line of the screen. + + Modifications to the internals of the screen is still possible through + this LineView however any modification will result in an undefined + behaviour. Don't do that. + + See BufferView. + """ + __slots__ = ("_line",) + def __init__(self, line): + self._line = line + + def __getitem__(self, x): + try: + return self._line[x] + except KeyError: + return self._line.default + +class BufferView: + """ + A read-only view of the screen. + + Modifications to the internals of the screen is still possible through + this BufferView however any modification will result in an undefined + behaviour. Don't do that. + + Any modification to the screen must be done through its method + (principally draw()) + """ + __slots__ = ("_buffer", "_screen") + def __init__(self, screen): + self._screen = screen + self._buffer = screen._buffer + + def __getitem__(self, y): + try: + line = self._buffer[y] + except KeyError: + line = Line(self._screen.default_char) + + return LineView(line) + + def __len__(self): + return self._screen.lines + class Screen: """ A screen is an in-memory matrix of characters that represents the @@ -388,7 +436,7 @@ def __init__(self, columns, lines): self.savepoints = [] self.columns = columns self.lines = lines - self.buffer = defaultdict(lambda: Line(self.default_char)) + self._buffer = defaultdict(lambda: Line(self.default_char)) self.dirty = set() self._default_style = CharStyle( @@ -402,12 +450,17 @@ def __init__(self, columns, lines): def __repr__(self): return ("{0}({1}, {2})".format(self.__class__.__name__, self.columns, self.lines)) + + @property + def buffer(self): + return BufferView(self) + def stats(self): """ Note: this is not part of the stable API so it may change between version of pyte. """ - buffer = self.buffer + buffer = self._buffer return BufferStats( empty=not bool(buffer), entries=len(buffer), @@ -429,7 +482,7 @@ def display(self): prev_y = -1 output = [] columns = self.columns - for y, line in sorted(self.buffer.items()): + for y, line in sorted(self._buffer.items()): empty_lines = y - (prev_y + 1) if empty_lines: output.extend([padding * columns] * empty_lines) @@ -485,7 +538,7 @@ def reset(self): :manpage:`xterm` -- we now know that. """ self.dirty.update(range(self.lines)) - self.buffer.clear() + self._buffer.clear() self.margins = None self.mode = set([mo.DECAWM, mo.DECTCEM]) @@ -541,7 +594,7 @@ def resize(self, lines=None, columns=None): self.restore_cursor() if columns < self.columns: - for line in self.buffer.values(): + for line in self._buffer.values(): for x in range(columns, self.columns): line.pop(x, None) @@ -612,7 +665,7 @@ def set_mode(self, *modes, **kwargs): # Mark all displayed characters as reverse. if mo.DECSCNM in modes: - for line in self.buffer.values(): + for line in self._buffer.values(): line.default = self.default_char for x in line: line[x].style = line[x].style._replace(reverse=True) @@ -650,7 +703,7 @@ def reset_mode(self, *modes, **kwargs): self.cursor_position() if mo.DECSCNM in modes: - for line in self.buffer.values(): + for line in self._buffer.values(): line.default = self.default_char for x in line: line[x].style = line[x].style._replace(reverse=False) @@ -706,7 +759,7 @@ def draw(self, data): # execution of self.draw() columns = self.columns cursor = self.cursor - buffer = self.buffer + buffer = self._buffer attrs = cursor.attrs mode = self.mode style = attrs.style @@ -837,7 +890,7 @@ def index(self): """ top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == bottom: - buffer = self.buffer + buffer = self._buffer pop = buffer.pop non_empty_y = sorted(buffer) @@ -860,7 +913,7 @@ def reverse_index(self): """ top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == top: - buffer = self.buffer + buffer = self._buffer pop = buffer.pop non_empty_y = sorted(buffer) @@ -954,9 +1007,9 @@ def insert_lines(self, count=None): if top <= self.cursor.y <= bottom: self.dirty.update(range(self.cursor.y, self.lines)) for y in range(bottom, self.cursor.y - 1, -1): - if y + count <= bottom and y in self.buffer: - self.buffer[y + count] = self.buffer[y] - self.buffer.pop(y, None) + if y + count <= bottom and y in self._buffer: + self._buffer[y + count] = self._buffer[y] + self._buffer.pop(y, None) self.carriage_return() @@ -976,10 +1029,10 @@ def delete_lines(self, count=None): self.dirty.update(range(self.cursor.y, self.lines)) for y in range(self.cursor.y, bottom + 1): if y + count <= bottom: - if y + count in self.buffer: - self.buffer[y] = self.buffer.pop(y + count) + if y + count in self._buffer: + self._buffer[y] = self._buffer.pop(y + count) else: - self.buffer.pop(y, None) + self._buffer.pop(y, None) self.carriage_return() @@ -994,7 +1047,7 @@ def insert_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - line = self.buffer[self.cursor.y] + line = self._buffer[self.cursor.y] for x in range(self.columns, self.cursor.x - 1, -1): new_x = x + count if new_x <= self.columns: @@ -1020,7 +1073,7 @@ def delete_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - line = self.buffer[self.cursor.y] + line = self._buffer[self.cursor.y] for x in range(self.cursor.x, self.columns): if x + count <= self.columns: line[x] = line.pop(x + count, self.default_char.copy()) @@ -1044,7 +1097,7 @@ def erase_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - write_data = self.buffer[self.cursor.y].write_data + write_data = self._buffer[self.cursor.y].write_data data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style @@ -1075,7 +1128,7 @@ def erase_in_line(self, how=0, private=False): elif how == 2: interval = range(self.columns) - write_data = self.buffer[self.cursor.y].write_data + write_data = self._buffer[self.cursor.y].write_data data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style @@ -1117,7 +1170,7 @@ def erase_in_display(self, how=0, *args, **kwargs): width = self.cursor.attrs.width style = self.cursor.attrs.style for y in interval: - line = self.buffer[y] + line = self._buffer[y] write_data = line.write_data for x in line: write_data(x, data, width, style) @@ -1286,7 +1339,7 @@ def alignment_display(self): style = self._default_style for y in range(self.lines): for x in range(self.columns): - self.buffer[y].write_data(x, "E", wcwidth("E"), style) + self._buffer[y].write_data(x, "E", wcwidth("E"), style) def select_graphic_rendition(self, *attrs): """Set display attributes. @@ -1495,7 +1548,7 @@ def after_event(self, event): :param str event: event name, for example ``"linefeed"``. """ if event in ["prev_page", "next_page"]: - for line in self.buffer.values(): + for line in self._buffer.values(): pop = line.pop for x in line: if x > self.columns: @@ -1533,7 +1586,7 @@ def index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == bottom: - self.history.top.append(self.buffer[top]) + self.history.top.append(self._buffer[top]) super(HistoryScreen, self).index() @@ -1542,7 +1595,7 @@ def reverse_index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == top: - self.history.bottom.append(self.buffer[bottom]) + self.history.bottom.append(self._buffer[bottom]) super(HistoryScreen, self).reverse_index() @@ -1557,15 +1610,15 @@ def prev_page(self): int(math.ceil(self.lines * self.history.ratio))) self.history.bottom.extendleft( - self.buffer[y] + self._buffer[y] for y in range(self.lines - 1, self.lines - mid - 1, -1)) self.history = self.history \ ._replace(position=self.history.position - mid) for y in range(self.lines - 1, mid - 1, -1): - self.buffer[y] = self.buffer[y - mid] + self._buffer[y] = self._buffer[y - mid] for y in range(mid - 1, -1, -1): - self.buffer[y] = self.history.top.pop() + self._buffer[y] = self.history.top.pop() self.dirty = set(range(self.lines)) @@ -1575,14 +1628,14 @@ def next_page(self): mid = min(len(self.history.bottom), int(math.ceil(self.lines * self.history.ratio))) - self.history.top.extend(self.buffer[y] for y in range(mid)) + self.history.top.extend(self._buffer[y] for y in range(mid)) self.history = self.history \ ._replace(position=self.history.position + mid) for y in range(self.lines - mid): - self.buffer[y] = self.buffer[y + mid] + self._buffer[y] = self._buffer[y + mid] for y in range(self.lines - mid, self.lines): - self.buffer[y] = self.history.bottom.popleft() + self._buffer[y] = self.history.bottom.popleft() self.dirty = set(range(self.lines)) diff --git a/tests/test_history.py b/tests/test_history.py index 00e668a..cec1f5a 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -28,13 +28,13 @@ def test_index(): line = screen.buffer[0] screen.index() assert screen.history.top - assert screen.history.top[-1] == line + assert screen.history.top[-1] == line._line # b) second index. line = screen.buffer[0] screen.index() assert len(screen.history.top) == 2 - assert screen.history.top[-1] == line + assert screen.history.top[-1] == line._line # c) rotation. for _ in range(screen.history.size * 2): @@ -62,13 +62,13 @@ def test_reverse_index(): line = screen.buffer[screen.lines-1] screen.reverse_index() assert screen.history.bottom - assert screen.history.bottom[0] == line + assert screen.history.bottom[0] == line._line # b) second index. line = screen.buffer[screen.lines-1] screen.reverse_index() assert len(screen.history.bottom) == 2 - assert screen.history.bottom[1] == line + assert screen.history.bottom[1] == line._line # c) rotation. for _ in range(screen.history.size * 2): diff --git a/tests/test_screen.py b/tests/test_screen.py index eb820f8..e65c4b0 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -33,7 +33,9 @@ def update(screen, lines, colored=[]): style = red_style else: style = base_style - screen.buffer[y].write_data(x, char, 1, style) + # Note: this hack is only for testing purposes. + # Modifying the screen's buffer is not allowed. + screen._buffer[y].write_data(x, char, 1, style) return screen From 5ae46bc8198bdeea05c0f0a5dc1ef847da100ecf Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Tue, 5 Jul 2022 17:24:04 -0300 Subject: [PATCH 22/54] Minor lookup prefetch. --- pyte/screens.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 6ecb982..6c96f06 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -667,8 +667,8 @@ def set_mode(self, *modes, **kwargs): if mo.DECSCNM in modes: for line in self._buffer.values(): line.default = self.default_char - for x in line: - line[x].style = line[x].style._replace(reverse=True) + for char in line.values(): + char.style = char.style._replace(reverse=True) self.select_graphic_rendition(7) # +reverse. @@ -705,8 +705,8 @@ def reset_mode(self, *modes, **kwargs): if mo.DECSCNM in modes: for line in self._buffer.values(): line.default = self.default_char - for x in line: - line[x].style = line[x].style._replace(reverse=False) + for char in line.values(): + char.style = char.style._replace(reverse=False) self.select_graphic_rendition(27) # -reverse. @@ -1338,8 +1338,9 @@ def alignment_display(self): self.dirty.update(range(self.lines)) style = self._default_style for y in range(self.lines): + line = self._buffer[y] for x in range(self.columns): - self._buffer[y].write_data(x, "E", wcwidth("E"), style) + line.write_data(x, "E", wcwidth("E"), style) def select_graphic_rendition(self, *attrs): """Set display attributes. From da66a7e6527716a99c026868dd08966b8e04aca7 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Tue, 5 Jul 2022 18:07:45 -0300 Subject: [PATCH 23/54] Do not unintentionally create empty lines Because screen._buffer is a defaultdict, an access to a non-existent element has the side effect --- pyte/screens.py | 55 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 6c96f06..1b91dd9 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -431,6 +431,8 @@ def default_char(self): style = self._default_style_reversed if mo.DECSCNM in self.mode else self._default_style return Char(" ", wcwidth(" "), style) + def default_line(self): + return Line(self.default_char) def __init__(self, columns, lines): self.savepoints = [] @@ -776,6 +778,7 @@ def draw(self, data): cursor_x = cursor.x cursor_y = cursor.y line = buffer[cursor_y] + write_data = line.write_data char_at = line.char_at for char in data: @@ -1047,7 +1050,14 @@ def insert_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - line = self._buffer[self.cursor.y] + line = self._buffer.get(self.cursor.y) + + # if there is no line (aka the line is empty), then don't do + # anything as insert_characters only moves the chars within + # the line but does not write anything new. + if not line: + return + for x in range(self.columns, self.cursor.x - 1, -1): new_x = x + count if new_x <= self.columns: @@ -1071,9 +1081,16 @@ def delete_characters(self, count=None): :param int count: number of characters to delete. """ self.dirty.add(self.cursor.y) + count = count or 1 + line = self._buffer.get(self.cursor.y) + + # if there is no line (aka the line is empty), then don't do + # anything as delete_characters only moves the chars within + # the line but does not write anything new except a default char + if not line: + return - line = self._buffer[self.cursor.y] for x in range(self.cursor.x, self.columns): if x + count <= self.columns: line[x] = line.pop(x + count, self.default_char.copy()) @@ -1159,18 +1176,24 @@ def erase_in_display(self, how=0, *args, **kwargs): parameter causing the stream to assume a ``0`` second parameter. """ if how == 0: - interval = range(self.cursor.y + 1, self.lines) + top, bottom = self.cursor.y + 1, self.lines elif how == 1: - interval = range(self.cursor.y) + top, bottom = 0, self.cursor.y elif how == 2 or how == 3: - interval = range(self.lines) + top, bottom = 0, self.lines - self.dirty.update(interval) + buffer = self._buffer + + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, top) # inclusive + end = bisect_left(non_empty_y, bottom, begin) # exclusive + + self.dirty.update(range(top, bottom)) data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style - for y in interval: - line = self._buffer[y] + for y in non_empty_y[begin:end]: + line = buffer[y] write_data = line.write_data for x in line: write_data(x, data, width, style) @@ -1587,7 +1610,7 @@ def index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == bottom: - self.history.top.append(self._buffer[top]) + self.history.top.append(self._buffer.get(top, self.default_line())) super(HistoryScreen, self).index() @@ -1596,7 +1619,7 @@ def reverse_index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == top: - self.history.bottom.append(self._buffer[bottom]) + self.history.bottom.append(self._buffer.get(bottom, self.default_line())) super(HistoryScreen, self).reverse_index() @@ -1611,8 +1634,10 @@ def prev_page(self): int(math.ceil(self.lines * self.history.ratio))) self.history.bottom.extendleft( - self._buffer[y] - for y in range(self.lines - 1, self.lines - mid - 1, -1)) + self._buffer.get(y, self.default_line()) + for y in range(self.lines - 1, self.lines - mid - 1, -1) + ) + self.history = self.history \ ._replace(position=self.history.position - mid) @@ -1629,7 +1654,11 @@ def next_page(self): mid = min(len(self.history.bottom), int(math.ceil(self.lines * self.history.ratio))) - self.history.top.extend(self._buffer[y] for y in range(mid)) + self.history.top.extend( + self._buffer.get(y, self.default_line()) + for y in range(mid) + ) + self.history = self.history \ ._replace(position=self.history.position + mid) From 1fd373a801f12cd022716505a075b8ed64efa07a Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 17:14:16 -0300 Subject: [PATCH 24/54] Add blankcs Stats --- pyte/screens.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 1b91dd9..0ff6097 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -103,6 +103,7 @@ class BufferStats(namedtuple("_BufferStats", [ "columns", "lines", "falses", + "blanks", "occupacy", "min", "max", @@ -123,16 +124,18 @@ def __repr__(self): if self.empty: return bstats + \ - "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f})\n{5}".format( + "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}), blanks: {5:> 3} ({6:.2f})\n{7}".format( self.entries, self.lines, self.occupacy, self.falses, self.falses/self.entries, + self.blanks, self.blanks/self.entries, "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) ) else: return bstats + \ - "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}); range: [{5: >3} - {6: >3}], len: {7: >3} ({8:.2f})\n{9}".format( + "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}), blanks: {5:> 3} ({6:.2f}); range: [{7: >3} - {8: >3}], len: {9: >3} ({10:.2f})\n{11}".format( self.entries, self.lines, self.occupacy, self.falses, self.falses/self.entries, + self.blanks, self.blanks/self.entries, self.min, self.max, self.span, self.span/self.lines, "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) ) @@ -469,6 +472,7 @@ def stats(self): columns=self.columns, lines=self.lines, falses=len([line for line in buffer.values() if not line]), + blanks=len([line for line in buffer.values() if all(char.data == " " for char in line.values())]), occupacy=len(buffer)/self.lines, min=min(buffer) if buffer else None, max=max(buffer) if buffer else None, From b8250ea4e88d13525890ea3b9e25165717eb8391 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 19:05:53 -0300 Subject: [PATCH 25/54] Use a space for padding screen.display screen.default_char is always the space character so instead of using screen.default_char for padding we use the space character directly. --- pyte/screens.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyte/screens.py b/pyte/screens.py index 0ff6097..9fb53e4 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -483,7 +483,10 @@ def stats(self): @property def display(self): """A :func:`list` of screen lines as unicode strings.""" - padding = self.default_char.data + # screen.default_char is always the space character + # We can skip the lookup of it and set the padding char + # directly + padding = " " prev_y = -1 output = [] From 8d715284339a8eb932a09df16631b4de23c3d230 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 19:10:15 -0300 Subject: [PATCH 26/54] Replace line's default style instead overwriting its char Because the default char of a line is always the space character, we don't need to overwrite it with screen.default_char, just we need to change its style. --- pyte/screens.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 9fb53e4..37d21ce 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -675,7 +675,7 @@ def set_mode(self, *modes, **kwargs): # Mark all displayed characters as reverse. if mo.DECSCNM in modes: for line in self._buffer.values(): - line.default = self.default_char + line.default.style = line.default.style._replace(reverse=True) for char in line.values(): char.style = char.style._replace(reverse=True) @@ -713,7 +713,7 @@ def reset_mode(self, *modes, **kwargs): if mo.DECSCNM in modes: for line in self._buffer.values(): - line.default = self.default_char + line.default.style = line.default.style._replace(reverse=False) for char in line.values(): char.style = char.style._replace(reverse=False) From 71ab12ab40ce2f03dfdec37a89f3ca3ec7fe5432 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 19:59:13 -0300 Subject: [PATCH 27/54] Fix a bug on index that top line was not removed. If the next line of the margin's top was not empty, the for-loop used that entry to override the top line working as expected. But when the next line of the top was empty, the top line was untouched so an explicit pop is required. A similar issue happen on reverse_index. Both bugs were covered due a side effect to iterating over screen.buffer: on each line lookup, if no such exist, a new line is added to the buffer. This added new line then was used to override the top on an index() call. --- pyte/screens.py | 14 ++++++++++++++ tests/test_screen.py | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/pyte/screens.py b/pyte/screens.py index 37d21ce..5450877 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -907,6 +907,13 @@ def index(self): begin = bisect_left(non_empty_y, top + 1) end = bisect_right(non_empty_y, bottom, begin) + # the top line must be unconditionally removed + # this pop is required because it may happen that + # the next line (top + 1) is empty and therefore + # the for-loop above didn't overwrite the line before + # (top + 1 - 1, aka top) + pop(top, None) + to_move = non_empty_y[begin:end] for y in to_move: buffer[y-1] = pop(y) @@ -930,6 +937,13 @@ def reverse_index(self): begin = bisect_left(non_empty_y, top) end = bisect_right(non_empty_y, bottom - 1, begin) + # the bottom line must be unconditionally removed + # this pop is required because it may happen that + # the previous line (bottom - 1) is empty and therefore + # the for-loop above didn't overwrite the line after + # (bottom - 1 + 1, aka bottom) + pop(bottom, None) + to_move = non_empty_y[begin:end] for y in reversed(to_move): buffer[y+1] = pop(y) diff --git a/tests/test_screen.py b/tests/test_screen.py index e65c4b0..19ca2b2 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -520,11 +520,14 @@ def test_carriage_return(): def test_index(): screen = update(pyte.Screen(2, 2), ["wo", "ot"], colored=[1]) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == ["wo", "ot"] # a) indexing on a row that isn't the last should just move # the cursor down. screen.index() assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == ["wo", "ot"] assert tolist(screen) == [ [Char("w"), Char("o")], [Char("o", fg="red"), Char("t", fg="red")] @@ -534,6 +537,7 @@ def test_index(): # create a new row at the bottom. screen.index() assert screen.cursor.y == 1 + assert screen.display == ["ot", " "] assert tolist(screen) == [ [Char("o", fg="red"), Char("t", fg="red")], [screen.default_char, screen.default_char] @@ -542,6 +546,8 @@ def test_index(): # c) same with margins screen = update(pyte.Screen(2, 5), ["bo", "sh", "th", "er", "oh"], colored=[1, 2]) + # note: margins are 0-based inclusive indexes for top and bottom + # however, set_margins are 1-based inclusive indexes screen.set_margins(2, 4) screen.cursor.y = 3 From 01f96bc6e0f1feb72561e8e4d2d61f766d71a463 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 20:05:58 -0300 Subject: [PATCH 28/54] BufferView not longer add new lines on iteration. --- pyte/screens.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 5450877..a8e4c43 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -348,9 +348,8 @@ def __init__(self, screen): self._buffer = screen._buffer def __getitem__(self, y): - try: - line = self._buffer[y] - except KeyError: + line = self._buffer.get(y) + if line is None: line = Line(self._screen.default_char) return LineView(line) From 70d4b15c12d8891530dff1f3b3f69c5034e38628 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Wed, 6 Jul 2022 20:47:19 -0300 Subject: [PATCH 29/54] Try to delete entries on erase instead of write spaces If the line's default char and the cursor's attributes are the same, instead of writing spaces into the line, delete the chars. This should be equivalent from user's perspective. This applies to erase_characters, erase_in_line and erase_in_display. This optimization reduced the number of false lines (lines without any char) and the number of blank lines (lines with only spaces). With less entries in the buffer, the rest of the iterations can take advantage of the sparsity of the buffer and process much less. --- pyte/screens.py | 91 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index a8e4c43..b169297 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1134,13 +1134,30 @@ def erase_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - write_data = self._buffer[self.cursor.y].write_data - data = self.cursor.attrs.data - width = self.cursor.attrs.width - style = self.cursor.attrs.style - for x in range(self.cursor.x, - min(self.cursor.x + count, self.columns)): - write_data(x, data, width, style) + line = self._buffer[self.cursor.y] + + # If the line's default char is equivalent to our cursor, overwriting + # a char in the line is equivalent to delete it if from the line + if line.default == self.cursor.attrs: + pop = line.pop + for x in range(self.cursor.x, + min(self.cursor.x + count, self.columns)): + pop(x, None) + + # the line may end up being empty, delete it from the buffer (*) + if not line: + del self._buffer[self.cursor.y] + + else: + write_data = line.write_data + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style + # a full range scan is required and not a sparse scan + # because we were asked to *write* on that full range + for x in range(self.cursor.x, + min(self.cursor.x + count, self.columns)): + write_data(x, data, width, style) def erase_in_line(self, how=0, private=False): """Erase a line in a specific way. @@ -1165,12 +1182,28 @@ def erase_in_line(self, how=0, private=False): elif how == 2: interval = range(self.columns) - write_data = self._buffer[self.cursor.y].write_data - data = self.cursor.attrs.data - width = self.cursor.attrs.width - style = self.cursor.attrs.style - for x in interval: - write_data(x, data, width, style) + line = self._buffer[self.cursor.y] + + # If the line's default char is equivalent to our cursor, overwriting + # a char in the line is equivalent to delete it if from the line + if line.default == self.cursor.attrs: + pop = line.pop + for x in interval: + pop(x, None) + + # the line may end up being empty, delete it from the buffer (*) + if not line: + del self._buffer[self.cursor.y] + + else: + write_data = line.write_data + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style + # a full range scan is required and not a sparse scan + # because we were asked to *write* on that full range + for x in interval: + write_data(x, data, width, style) def erase_in_display(self, how=0, *args, **kwargs): """Erases display in a specific way. @@ -1209,14 +1242,32 @@ def erase_in_display(self, how=0, *args, **kwargs): end = bisect_left(non_empty_y, bottom, begin) # exclusive self.dirty.update(range(top, bottom)) - data = self.cursor.attrs.data - width = self.cursor.attrs.width - style = self.cursor.attrs.style - for y in non_empty_y[begin:end]: - line = buffer[y] + + # Remove the lines from the buffer as this is equivalent + # to overwrite each char in them with the space character + # (screen.default_char). + # If a deleted line is then requested, a new line will + # be added with screen.default_char as its default char + if self.default_char == self.cursor.attrs: + for y in non_empty_y[begin:end]: + del buffer[y] + + else: write_data = line.write_data - for x in line: - write_data(x, data, width, style) + data = self.cursor.attrs.data + width = self.cursor.attrs.width + style = self.cursor.attrs.style + for y in non_empty_y[begin:end]: + line = buffer[y] + write_data = line.write_data + # TODO: note that this may not be entirely correct as + # 'for x in line' iterates over the non-empty chars + # of the line, changing their data when write_data() + # is called. + # But this means that any empty char in the line + # is never touch, in particular, its attributes. + for x in line: + write_data(x, data, width, style) if how == 0 or how == 1: self.erase_in_line(how) From 4a15d3ac3157ec3ddd95ae2e81f1c6b1cdd18476 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Thu, 7 Jul 2022 21:22:59 -0300 Subject: [PATCH 30/54] Impl prev_page/next_page with sparse iteration We avoid a full scan and instead we do a sparse iteration. This also avoids adding empty lines into the buffer as real entries when a non-existing entry has the same effect and it consumes less memory. --- pyte/screens.py | 96 ++++++++++++++-- tests/test_history.py | 228 +++++++++++++++++++++++++++++++++++++ tests/test_input_output.py | 22 ++++ 3 files changed, 338 insertions(+), 8 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index b169297..a27a272 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1704,18 +1704,62 @@ def prev_page(self): mid = min(len(self.history.top), int(math.ceil(self.lines * self.history.ratio))) + buffer = self._buffer + pop = buffer.pop + self.history.bottom.extendleft( - self._buffer.get(y, self.default_line()) + buffer.get(y, self.default_line()) for y in range(self.lines - 1, self.lines - mid - 1, -1) ) self.history = self.history \ ._replace(position=self.history.position - mid) - for y in range(self.lines - 1, mid - 1, -1): - self._buffer[y] = self._buffer[y - mid] + non_empty_y = sorted(buffer) + end = bisect_left(non_empty_y, self.lines - mid) + + to_move = reversed(non_empty_y[:end]) + + # to_move + # |---------------| + # 0 1 x 3 4 5 mid = 2 (x means empty) + # + # 0 1 0 1 4 3 (first for-loop without the inner loop: the "4" is wrong) + # + # 0 1 0 1 x 3 (first for-loop with the inner loop: the "4" is removed) + # + # P P 0 1 x 3 (after third for-loop, P are from history.top) + next_y = self.lines - mid + for y in to_move: + # Notice how if (y + 1) == (next_y) then you know + # that no empty lines are in between this y and the next one + # and therefore the range() loop gets empty. + # In other cases, (y + 1) < (next_y) + for z in range(y + 1 + mid, next_y + mid): + pop(z, None) + + # it may look weird but the current "y" is the "next_y" + # of the next iteration because we are iterating to_move + # backwards + next_y = y + buffer[y + mid] = buffer[y] + + # between the last moved line and the begin of the page + # we may have lines that should be emptied + for z in range(0 + mid, next_y + mid): + pop(z, None) + for y in range(mid - 1, -1, -1): - self._buffer[y] = self.history.top.pop() + line = self.history.top.pop() + if line: + # note: empty lines are not added as they are + # the default for non-existent entries in buffer + buffer[y] = line + else: + # because empty lines are not added we need to ensure + # that the old lines in that position become empty + # anyways (aka, we remove the old ones) + pop(y, None) self.dirty = set(range(self.lines)) @@ -1725,18 +1769,54 @@ def next_page(self): mid = min(len(self.history.bottom), int(math.ceil(self.lines * self.history.ratio))) + buffer = self._buffer + pop = buffer.pop + self.history.top.extend( - self._buffer.get(y, self.default_line()) + buffer.get(y, self.default_line()) for y in range(mid) ) self.history = self.history \ ._replace(position=self.history.position + mid) - for y in range(self.lines - mid): - self._buffer[y] = self._buffer[y + mid] + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, mid) + + to_move = non_empty_y[begin:] + + # to_move + # |---------------| + # 0 1 2 x 4 5 mid = 2 + # + # 2 1 4 5 4 5 + # + # 2 3 4 5 P P (final result) + + prev_y = mid - 1 + for y in to_move: + # Notice how if (prev_y + 1) == (y) then you know + # that no empty lines are in between and therefore + # the range() loop gets empty. + # In other cases, (prev_y + 1) > (y) + for z in range(prev_y + 1 - mid, y - mid): + pop(z, None) + + prev_y = y + buffer[y - mid] = buffer[y] + + for z in range(prev_y + 1 - mid, self.lines - mid): + pop(z, None) + for y in range(self.lines - mid, self.lines): - self._buffer[y] = self.history.bottom.popleft() + line = self.history.bottom.popleft() + if line: + buffer[y] = line + else: + # because empty lines are not added we need to ensure + # that the old lines in that position become empty + # anyways (aka, we remove the old ones) + pop(y, None) self.dirty = set(range(self.lines)) diff --git a/tests/test_history.py b/tests/test_history.py index cec1f5a..4b2fc50 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -327,6 +327,130 @@ def test_prev_page(): consistency_asserts(screen) +def test_prev_page_large_sparse(): + # like test_prev_page, this test does the same checks + # but uses a larger screen and it does not write on every + # line. + # Because screen.buffer is optimized to not have entries + # for empty lines, this setup may uncover bugs that + # test_prev_page cannot + screen = pyte.HistoryScreen(4, 8, history=16) + screen.set_mode(mo.LNM) + + assert screen.history.position == 16 + + # Filling the screen with line numbers but only + # if they match the following sequence. + # This is to leave some empty lines in between + # to test the sparsity of the buffer. + FB = [2, 5, 8, 13, 18] + for idx in range(19): + if idx in FB: + screen.draw(str(idx)) + screen.linefeed() + + assert screen.history.top + assert not screen.history.bottom + assert screen.history.position == 16 + assert screen.display == [ + " ", + "13 ", + " ", + " ", + " ", + " ", + "18 ", + " ", + ] + consistency_asserts(screen) + + assert chars(screen.history.top, screen.columns) == [ + " ", + " ", + "2 ", + " ", + " ", + "5 ", + " ", + " ", + "8 ", + " ", + " ", + " ", + ] + + # a) first page up. + screen.prev_page() + assert screen.history.position == 12 + assert len(screen.buffer) == screen.lines + assert screen.display == [ + "8 ", + " ", + " ", + " ", + " ", + "13 ", + " ", + " ", + ] + consistency_asserts(screen) + + assert chars(screen.history.top, screen.columns) == [ + " ", + " ", + "2 ", + " ", + " ", + "5 ", + " ", + " ", + ] + + assert len(screen.history.bottom) == 4 + assert chars(screen.history.bottom, screen.columns) == [ + " ", + " ", + "18 ", + " ", + ] + + # b) second page up. + screen.prev_page() + assert screen.history.position == 8 + assert len(screen.buffer) == screen.lines + assert screen.display == [ + " ", + "5 ", + " ", + " ", + "8 ", + " ", + " ", + " ", + ] + consistency_asserts(screen) + + assert len(screen.history.bottom) == 8 + assert chars(screen.history.bottom, screen.columns) == [ + " ", + "13 ", + " ", + " ", + " ", + " ", + "18 ", + " ", + ] + + # c) third page up? + # TODO this seems to not work as the remaining lines in the history + # are not moved into the buffer. This is because the condition + # + # if self.history.position > self.lines and self.history.top: + # .... + # This bug/issue is present on 0.8.1 + + def test_next_page(): screen = pyte.HistoryScreen(5, 5, history=50) screen.set_mode(mo.LNM) @@ -401,6 +525,110 @@ def test_next_page(): ] consistency_asserts(screen) +def test_next_page_large_sparse(): + screen = pyte.HistoryScreen(5, 8, history=16) + screen.set_mode(mo.LNM) + + assert screen.history.position == 16 + + # Filling the screen with line numbers but only + # if they match the following sequence. + # This is to leave some empty lines in between + # to test the sparsity of the buffer. + FB = [2, 5, 8, 13, 18] + for idx in range(19): + if idx in FB: + screen.draw(str(idx)) + screen.linefeed() + + assert screen.history.top + assert not screen.history.bottom + assert screen.history.position == 16 + assert screen.display == [ + " ", + "13 ", + " ", + " ", + " ", + " ", + "18 ", + " ", + ] + consistency_asserts(screen) + + # a) page up -- page down. + screen.prev_page() + screen.next_page() + assert screen.history.top + assert not screen.history.bottom + assert screen.history.position == 16 + assert screen.display == [ + " ", + "13 ", + " ", + " ", + " ", + " ", + "18 ", + " ", + ] + consistency_asserts(screen) + + # b) double page up -- page down. + screen.prev_page() + screen.prev_page() + screen.next_page() + assert screen.history.position == 12 + assert screen.history.top + assert chars(screen.history.bottom, screen.columns) == [ + " ", + " ", + "18 ", + " ", + ] + assert chars(screen.history.top, screen.columns) == [ + " ", + " ", + "2 ", + " ", + " ", + "5 ", + " ", + " ", + ] + + assert screen.display == [ + "8 ", + " ", + " ", + " ", + " ", + "13 ", + " ", + " ", + ] + consistency_asserts(screen) + + # c) page down -- double page up -- double page down + screen.next_page() + screen.prev_page() + screen.prev_page() + screen.next_page() + screen.next_page() + assert screen.history.position == 16 + assert len(screen.buffer) == screen.lines + assert screen.display == [ + " ", + "13 ", + " ", + " ", + " ", + " ", + "18 ", + " ", + ] + consistency_asserts(screen) + def test_ensure_width(monkeypatch): screen = pyte.HistoryScreen(5, 5, history=50) diff --git a/tests/test_input_output.py b/tests/test_input_output.py index ab6535d..9838826 100644 --- a/tests/test_input_output.py +++ b/tests/test_input_output.py @@ -26,3 +26,25 @@ def test_input_output(name): stream.feed(input) assert screen.display == output consistency_asserts(screen) + +@pytest.mark.parametrize("name", [ + "cat-gpl3", "find-etc", "htop", "ls", "mc", "top", "vi" +]) +def test_input_output_history(name): + with open(os.path.join(captured_dir, name + ".input"), "rb") as handle: + input = handle.read() + + with open(os.path.join(captured_dir, name + ".output")) as handle: + output = json.load(handle) + + screen = pyte.HistoryScreen(80, 24, history=72) + stream = pyte.ByteStream(screen) + stream.feed(input) + screen.prev_page() + screen.prev_page() + screen.prev_page() + screen.next_page() + screen.next_page() + screen.next_page() + assert screen.display == output + consistency_asserts(screen) From 6b4f088c936260c6b8d12d44b31676d0efc54ada Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Thu, 7 Jul 2022 21:33:30 -0300 Subject: [PATCH 31/54] Make lines in history a LineView and make Line raise on non-key Instead of storing Line objects in the history's top and bottom lists, store LineView of them. This makes them read-only and allows us to make Line a full dict without an overloaded __missing__. Skipping __missing__ ensures that the code of Screen does not access an x position that does not exist by mistake. --- pyte/screens.py | 41 +++++++++++++++++++++-------------------- tests/test_history.py | 10 +++++----- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index a27a272..def6dbc 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -256,24 +256,10 @@ def __init__(self, x, y, attrs): class Line(dict): - """A :func:`dict` with a static default value representing a line of the screen. - - Unlike :func:`collections.defaultdict` this implementation does not - implicitly update the mapping when queried with a missing key. - - >>> d = Line(42) - >>> d["foo"] - 42 - >>> d - {} - """ __slots__ = ('default', ) def __init__(self, default): self.default = default - def __missing__(self, key): - return self.default - def write_data(self, x, data, width, style): """ Update the char at the position x with the new data, width and style. @@ -331,6 +317,19 @@ def __getitem__(self, x): except KeyError: return self._line.default + def __eq__(self, other): + if not isinstance(other, LineView): + raise TypeError() + + return self._line == other._line + + def __ne__(self, other): + if not isinstance(other, LineView): + raise TypeError() + + return self._line == other._line + + class BufferView: """ A read-only view of the screen. @@ -1681,7 +1680,7 @@ def index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == bottom: - self.history.top.append(self._buffer.get(top, self.default_line())) + self.history.top.append(self.buffer[top]) super(HistoryScreen, self).index() @@ -1690,7 +1689,7 @@ def reverse_index(self): top, bottom = self.margins or Margins(0, self.lines - 1) if self.cursor.y == top: - self.history.bottom.append(self._buffer.get(bottom, self.default_line())) + self.history.bottom.append(self.buffer[bottom]) super(HistoryScreen, self).reverse_index() @@ -1704,11 +1703,12 @@ def prev_page(self): mid = min(len(self.history.top), int(math.ceil(self.lines * self.history.ratio))) + bufferview = self.buffer buffer = self._buffer pop = buffer.pop self.history.bottom.extendleft( - buffer.get(y, self.default_line()) + bufferview[y] for y in range(self.lines - 1, self.lines - mid - 1, -1) ) @@ -1750,7 +1750,7 @@ def prev_page(self): pop(z, None) for y in range(mid - 1, -1, -1): - line = self.history.top.pop() + line = self.history.top.pop()._line if line: # note: empty lines are not added as they are # the default for non-existent entries in buffer @@ -1769,11 +1769,12 @@ def next_page(self): mid = min(len(self.history.bottom), int(math.ceil(self.lines * self.history.ratio))) + bufferview = self.buffer buffer = self._buffer pop = buffer.pop self.history.top.extend( - buffer.get(y, self.default_line()) + bufferview[y] for y in range(mid) ) @@ -1809,7 +1810,7 @@ def next_page(self): pop(z, None) for y in range(self.lines - mid, self.lines): - line = self.history.bottom.popleft() + line = self.history.bottom.popleft()._line if line: buffer[y] = line else: diff --git a/tests/test_history.py b/tests/test_history.py index 4b2fc50..93c6058 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -28,13 +28,13 @@ def test_index(): line = screen.buffer[0] screen.index() assert screen.history.top - assert screen.history.top[-1] == line._line + assert screen.history.top[-1] == line # b) second index. line = screen.buffer[0] screen.index() assert len(screen.history.top) == 2 - assert screen.history.top[-1] == line._line + assert screen.history.top[-1] == line # c) rotation. for _ in range(screen.history.size * 2): @@ -62,13 +62,13 @@ def test_reverse_index(): line = screen.buffer[screen.lines-1] screen.reverse_index() assert screen.history.bottom - assert screen.history.bottom[0] == line._line + assert screen.history.bottom[0] == line # b) second index. line = screen.buffer[screen.lines-1] screen.reverse_index() assert len(screen.history.bottom) == 2 - assert screen.history.bottom[1] == line._line + assert screen.history.bottom[1] == line # c) rotation. for _ in range(screen.history.size * 2): @@ -656,7 +656,7 @@ def test_ensure_width(monkeypatch): stream.feed(ctrl.ESC + "P") # Inequality because we have an all-empty last line. - assert all(len(l) <= 3 for l in screen.history.bottom) + assert all(len(l._line) <= 3 for l in screen.history.bottom) assert screen.display == [ "001", # 18 "001", # 19 From 13cf059dc9ca75ebaa44872920f578c36c4d5e0c Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 18:41:07 -0300 Subject: [PATCH 32/54] Test sparsity on index and reverse_index --- tests/test_screen.py | 200 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 198 insertions(+), 2 deletions(-) diff --git a/tests/test_screen.py b/tests/test_screen.py index 19ca2b2..51e2206 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -21,7 +21,7 @@ def __init__(self, data=" ", fg="default", bg="default", bold=False, italics=Fal # Test helpers. -def update(screen, lines, colored=[]): +def update(screen, lines, colored=[], write_spaces=True): """Updates a given screen object with given lines, colors each line from ``colored`` in "red" and returns the modified screen. """ @@ -35,7 +35,11 @@ def update(screen, lines, colored=[]): style = base_style # Note: this hack is only for testing purposes. # Modifying the screen's buffer is not allowed. - screen._buffer[y].write_data(x, char, 1, style) + if char == ' ' and not write_spaces: + # skip, leave the default char in the screen + pass + else: + screen._buffer[y].write_data(x, char, 1, style) return screen @@ -604,6 +608,105 @@ def test_index(): consistency_asserts(screen) +def test_index_sparse(): + screen = update(pyte.Screen(5, 5), + ["wo ", + " ", + " o t ", + " ", + "x z", + ], + colored=[2], + write_spaces=False) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + "wo ", + " ", + " o t ", + " ", + "x z", + ] + + # a) indexing on a row that isn't the last should just move + # the cursor down. + screen.index() + assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == [ + "wo ", + " ", + " o t ", + " ", + "x z", + ] + assert tolist(screen) == [ + [Char("w"), Char("o"),] + [screen.default_char] * 3, + [screen.default_char] * 5, + [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], + [screen.default_char] * 5, + [Char("x")] + [screen.default_char] * 3 + [Char("z")], + ] + + # b) indexing on the last row should push everything up and + # create a new row at the bottom. + screen.index() + screen.index() + screen.index() + screen.index() + assert screen.cursor.y == 4 + assert screen.display == [ + " ", + " o t ", + " ", + "x z", + " ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], + [screen.default_char] * 5, + [Char("x")] + [screen.default_char] * 3 + [Char("z")], + [screen.default_char] * 5, + ] + + # again + screen.index() + assert screen.cursor.y == 4 + assert screen.display == [ + " o t ", + " ", + "x z", + " ", + " ", + ] + assert tolist(screen) == [ + [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], + [screen.default_char] * 5, + [Char("x")] + [screen.default_char] * 3 + [Char("z")], + [screen.default_char] * 5, + [screen.default_char] * 5, + ] + + # leave the screen cleared + screen.index() + screen.index() + screen.index() + assert (screen.cursor.y, screen.cursor.x) == (4, 0) + assert screen.display == [ + " ", + " ", + " ", + " ", + " ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + ] + + def test_reverse_index(): screen = update(pyte.Screen(2, 2), ["wo", "ot"], colored=[0]) @@ -683,6 +786,99 @@ def test_reverse_index(): consistency_asserts(screen) +def test_reverse_index_sparse(): + screen = update(pyte.Screen(5, 5), + ["wo ", + " ", + " o t ", + " ", + "x z", + ], + colored=[2], + write_spaces=False) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + "wo ", + " ", + " o t ", + " ", + "x z", + ] + + # a) reverse indexing on the first row should push rows down + # and create a new row at the top. + screen.reverse_index() + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + " ", + "wo ", + " ", + " o t ", + " ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [Char("w"), Char("o"),] + [screen.default_char] * 3, + [screen.default_char] * 5, + [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], + [screen.default_char] * 5, + ] + + # again + screen.reverse_index() + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + " ", + " ", + "wo ", + " ", + " o t ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [screen.default_char] * 5, + [Char("w"), Char("o"),] + [screen.default_char] * 3, + [screen.default_char] * 5, + [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], + ] + + # again + screen.reverse_index() + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + " ", + " ", + " ", + "wo ", + " ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + [Char("w"), Char("o"),] + [screen.default_char] * 3, + [screen.default_char] * 5, + ] + + # leave the screen cleared + screen.reverse_index() + screen.reverse_index() + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [ + " ", + " ", + " ", + " ", + " ", + ] + assert tolist(screen) == [ + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + [screen.default_char] * 5, + ] + def test_linefeed(): screen = update(pyte.Screen(2, 2), ["bo", "sh"], [None, None]) screen.set_mode(mo.LNM) From a4e7ed59f974b63228e957dd8ae9b5af0c48c8f4 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 19:11:56 -0300 Subject: [PATCH 33/54] Test sparsity of insert_lines/delete_lines (fix bug on delete_lines) --- pyte/screens.py | 5 +-- tests/test_screen.py | 101 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index def6dbc..1fbdf15 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1050,9 +1050,8 @@ def delete_lines(self, count=None): if top <= self.cursor.y <= bottom: self.dirty.update(range(self.cursor.y, self.lines)) for y in range(self.cursor.y, bottom + 1): - if y + count <= bottom: - if y + count in self._buffer: - self._buffer[y] = self._buffer.pop(y + count) + if y + count <= bottom and y + count in self._buffer: + self._buffer[y] = self._buffer.pop(y + count) else: self._buffer.pop(y, None) diff --git a/tests/test_screen.py b/tests/test_screen.py index 51e2206..fe7c972 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1059,6 +1059,37 @@ def test_insert_lines(): ] consistency_asserts(screen) + + screen.insert_lines(1) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", "sam"] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [Char("s"), Char("a"), Char("m")] + ] + consistency_asserts(screen) + + screen.insert_lines(1) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " "] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + ] + consistency_asserts(screen) + + screen.insert_lines(1) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " "] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + ] + consistency_asserts(screen) + screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[1]) screen.insert_lines(2) @@ -1071,6 +1102,54 @@ def test_insert_lines(): ] consistency_asserts(screen) + screen = update(pyte.Screen(3, 5), [ + "sam", + "", # an empty string will be interpreted as a full empty line + "foo", + "bar", + "baz" + ], + colored=[2, 3]) + + screen.insert_lines(2) + + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", "sam", " ", "foo"] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [Char("s"), Char("a"), Char("m")], + [screen.default_char] * 3, + [Char("f", fg="red"), Char("o", fg="red"), Char("o", fg="red")], + ] + consistency_asserts(screen) + + screen.insert_lines(1) + + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " ", "sam", " "] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + [Char("s"), Char("a"), Char("m")], + [screen.default_char] * 3, + ] + consistency_asserts(screen) + + screen.insert_lines(1) + + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " ", " ", "sam"] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + [Char("s"), Char("a"), Char("m")], + ] + consistency_asserts(screen) + # b) with margins screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], colored=[2, 3]) @@ -1180,6 +1259,28 @@ def test_delete_lines(): ] consistency_asserts(screen) + screen.delete_lines(0) + + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " "] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + ] + consistency_asserts(screen) + + screen.delete_lines(0) + + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" ", " ", " "] + assert tolist(screen) == [ + [screen.default_char] * 3, + [screen.default_char] * 3, + [screen.default_char] * 3, + ] + consistency_asserts(screen) + # b) with margins screen = update(pyte.Screen(3, 5), ["sam", "is ", "foo", "bar", "baz"], colored=[2, 3]) From 3056742c84c72bdb97b33b129180e2c710706eaa Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 19:28:04 -0300 Subject: [PATCH 34/54] Test sparsity on insert_characters/delete_characters/erase_characters --- tests/test_screen.py | 117 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/tests/test_screen.py b/tests/test_screen.py index fe7c972..4a01520 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1385,6 +1385,31 @@ def test_insert_characters(): Char("b"), screen.default_char, screen.default_char ] + assert screen.display == [" s", "is ", "f o", "b "] + + # insert 1 at the begin of the previously edited line + screen.cursor.y, screen.cursor.x = 3, 0 + screen.insert_characters(1) + assert tolist(screen)[3] == [ + screen.default_char, Char("b"), screen.default_char, + ] + + # insert before the end of the line + screen.cursor.y, screen.cursor.x = 3, 2 + screen.insert_characters(1) + assert tolist(screen)[3] == [ + screen.default_char, Char("b"), screen.default_char, + ] + + # insert enough to push outside the screen the remaining char + screen.cursor.y, screen.cursor.x = 3, 0 + screen.insert_characters(2) + assert tolist(screen)[3] == [ + screen.default_char, screen.default_char, screen.default_char, + ] + + assert screen.display == [" s", "is ", "f o", " "] + # d) 0 is 1 screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[0]) @@ -1404,6 +1429,46 @@ def test_insert_characters(): ] + # ! extreme cases. + screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) + screen.cursor.x = 1 + screen.insert_characters(3) + assert (screen.cursor.y, screen.cursor.x) == (0, 1) + assert screen.display == ["1 2"] + assert tolist(screen)[0] == [ + Char("1", fg="red"), + screen.default_char, + screen.default_char, + screen.default_char, + Char("2", fg="red"), + ] + consistency_asserts(screen) + + screen.insert_characters(1) + assert (screen.cursor.y, screen.cursor.x) == (0, 1) + assert screen.display == ["1 "] + assert tolist(screen)[0] == [ + Char("1", fg="red"), + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + ] + consistency_asserts(screen) + + screen.cursor.x = 0 + screen.insert_characters(5) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" "] + assert tolist(screen)[0] == [ + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + ] + consistency_asserts(screen) + def test_delete_characters(): screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[0]) screen.delete_characters(2) @@ -1427,6 +1492,20 @@ def test_delete_characters(): assert screen.display == ["m ", "i ", "fo "] consistency_asserts(screen) + # try to erase spaces + screen.cursor.y, screen.cursor.x = 1, 1 + screen.delete_characters(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 1) + assert screen.display == ["m ", "i ", "fo "] + consistency_asserts(screen) + + # try to erase a whole line + screen.cursor.y, screen.cursor.x = 1, 0 + screen.delete_characters(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == ["m ", " ", "fo "] + consistency_asserts(screen) + # ! extreme cases. screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.cursor.x = 1 @@ -1469,6 +1548,18 @@ def test_delete_characters(): ] consistency_asserts(screen) + screen.delete_characters(2) + assert (screen.cursor.y, screen.cursor.x) == (0, 0) + assert screen.display == [" "] + assert tolist(screen)[0] == [ + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char + ] + consistency_asserts(screen) + def test_erase_character(): screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[0]) @@ -1495,6 +1586,20 @@ def test_erase_character(): assert screen.display == [" m", "i ", "fo "] consistency_asserts(screen) + # erase the same erased char as before + screen.cursor.y, screen.cursor.x = 1, 1 + screen.erase_characters(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 1) + assert screen.display == [" m", "i ", "fo "] + consistency_asserts(screen) + + # erase the whole line + screen.cursor.y, screen.cursor.x = 1, 0 + screen.erase_characters(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == [" m", " ", "fo "] + consistency_asserts(screen) + # ! extreme cases. screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.cursor.x = 1 @@ -1537,6 +1642,18 @@ def test_erase_character(): ] consistency_asserts(screen) + screen.cursor.x = 2 + screen.erase_characters(4) + assert (screen.cursor.y, screen.cursor.x) == (0, 2) + assert screen.display == [" "] + assert tolist(screen)[0] == [ + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + ] + consistency_asserts(screen) def test_erase_in_line(): screen = update(pyte.Screen(5, 5), From f96ab6b5693fd155ef9598593972685f6846e10c Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 19:50:50 -0300 Subject: [PATCH 35/54] Impl repr of a Char --- pyte/screens.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyte/screens.py b/pyte/screens.py index 1fbdf15..90673d3 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -237,6 +237,26 @@ def __ne__(self, other): return any(getattr(self, name) != getattr(other, name) for name in self._fields) + def __repr__(self): + r = "'%s'" % self.data + attrs = [] + if self.fg != "default": + attrs.append("fg=%s" % self.fg) + if self.bg != "default": + attrs.append("bg=%s" % self.bg) + + for attrname in ['bold', 'italics', 'underscore', + 'strikethrough', 'reverse', 'blink']: + val = getattr(self, attrname) + if val: + attrs.append("%s=%s" % (attrname, val)) + + if attrs: + r += " (" + (", ".join(attrs)) + ")" + + return r + + class Cursor: """Screen cursor. From 70763b6b63b56c894af8de19f106c9a07e656aed Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 19:51:13 -0300 Subject: [PATCH 36/54] Extend erase_* meth tests for sparsity and cursor attr usage (fixed a bug) --- pyte/screens.py | 1 - tests/test_screen.py | 145 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 90673d3..ed201ff 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1271,7 +1271,6 @@ def erase_in_display(self, how=0, *args, **kwargs): del buffer[y] else: - write_data = line.write_data data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style diff --git a/tests/test_screen.py b/tests/test_screen.py index 4a01520..5af2e72 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1600,6 +1600,33 @@ def test_erase_character(): assert screen.display == [" m", " ", "fo "] consistency_asserts(screen) + # erase 2 chars of an already-empty line with a cursor having a different + # attribute + screen.select_graphic_rendition(31) # red foreground + screen.cursor.y, screen.cursor.x = 1, 0 + screen.erase_characters(2) + assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == [" m", " ", "fo "] + assert tolist(screen)[1] == [ + Char(" ", fg='red'), + Char(" ", fg='red'), + screen.default_char + ] + consistency_asserts(screen) + + # erase 1 chars of a non-empty line with a cursor having a different + # attribute + screen.cursor.y, screen.cursor.x = 2, 1 + screen.erase_characters(1) + assert (screen.cursor.y, screen.cursor.x) == (2, 1) + assert screen.display == [" m", " ", "f "] + assert tolist(screen)[2] == [ + Char("f"), + Char(" ", fg='red'), + screen.default_char + ] + consistency_asserts(screen) + # ! extreme cases. screen = update(pyte.Screen(5, 1), ["12345"], colored=[0]) screen.cursor.x = 1 @@ -1661,7 +1688,7 @@ def test_erase_in_line(): "s foo", "but a", "re yo", - "u? "], colored=[0]) + "u? "], colored=[0, 1]) screen.cursor_position(1, 3) # a) erase from cursor to the end of line @@ -1681,6 +1708,59 @@ def test_erase_in_line(): ] consistency_asserts(screen) + # erase from cursor to the end of line (again, same place) + screen.erase_in_line(0) + assert (screen.cursor.y, screen.cursor.x) == (0, 2) + assert screen.display == ["sa ", + "s foo", + "but a", + "re yo", + "u? "] + assert tolist(screen)[0] == [ + Char("s", fg="red"), + Char("a", fg="red"), + screen.default_char, + screen.default_char, + screen.default_char + ] + consistency_asserts(screen) + + # erase from cursor to the end of line (again but from the middle of a line)) + screen.cursor.y = 1 + screen.erase_in_line(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 2) + assert screen.display == ["sa ", + "s ", + "but a", + "re yo", + "u? "] + assert tolist(screen)[1] == [ + Char("s", fg="red"), + Char(" ", fg="red"), # this space comes from the setup, not from the erase + screen.default_char, + screen.default_char, + screen.default_char + ] + consistency_asserts(screen) + + # erase from cursor to the end of line erasing the whole line + screen.cursor.x = 0 + screen.erase_in_line(0) + assert (screen.cursor.y, screen.cursor.x) == (1, 0) + assert screen.display == ["sa ", + " ", + "but a", + "re yo", + "u? "] + assert tolist(screen)[1] == [ + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char, + screen.default_char + ] + consistency_asserts(screen) + # b) erase from the beginning of the line to the cursor screen = update(screen, ["sam i", @@ -1688,6 +1768,8 @@ def test_erase_in_line(): "but a", "re yo", "u? "], colored=[0]) + screen.cursor.x = 2 + screen.cursor.y = 0 screen.erase_in_line(1) assert (screen.cursor.y, screen.cursor.x) == (0, 2) assert screen.display == [" i", @@ -1721,6 +1803,37 @@ def test_erase_in_line(): assert tolist(screen)[0] == [screen.default_char] * 5 consistency_asserts(screen) + # d) erase with a non-default attributes cursor + screen.select_graphic_rendition(31) # red foreground + + screen.cursor.y = 1 + screen.erase_in_line(2) + assert (screen.cursor.y, screen.cursor.x) == (1, 2) + assert screen.display == [" ", + " ", + "but a", + "re yo", + "u? "] + assert tolist(screen)[1] == [Char(" ", fg="red")] * 5 + consistency_asserts(screen) + + screen.cursor.y = 2 + screen.erase_in_line(1) + assert (screen.cursor.y, screen.cursor.x) == (2, 2) + assert screen.display == [" ", + " ", + " a", + "re yo", + "u? "] + assert tolist(screen)[2] == [ + Char(" ", fg="red"), + Char(" ", fg="red"), + Char(" ", fg="red"), + screen.default_char, + Char("a"), + ] + + consistency_asserts(screen) def test_erase_in_display(): screen = update(pyte.Screen(5, 5), @@ -1834,6 +1947,36 @@ def test_erase_in_display(): " "] consistency_asserts(screen) + # erase from the beginning of the display to the cursor, + # including it, but with the cursor having a non-default attribute + screen = update(screen, + ["sam i", + "s foo", + "but a", + "re yo", + "u? "], colored=[2, 3]) + + screen.cursor.x = 2 + screen.cursor.y = 2 + screen.select_graphic_rendition(31) # red foreground + screen.erase_in_display(1) + assert (screen.cursor.y, screen.cursor.x) == (2, 2) + assert screen.display == [" ", + " ", + " a", + "re yo", + "u? "] + assert tolist(screen)[:3] == [ + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red"), + Char(" ", fg="red"), + Char(" ", fg="red"), + Char(" ", fg="red"), + Char("a", fg="red")], + ] + consistency_asserts(screen) + def test_cursor_up(): screen = pyte.Screen(10, 10) From 51e79a6ca9dffd577e28c05082b11e0f04fb1093 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 20:08:51 -0300 Subject: [PATCH 37/54] Make erase_in_display conformant (test with non-default cursor attrs) --- pyte/screens.py | 25 +++++++++++++------------ tests/test_screen.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index ed201ff..a542f3b 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1255,18 +1255,25 @@ def erase_in_display(self, how=0, *args, **kwargs): buffer = self._buffer - non_empty_y = sorted(buffer) - begin = bisect_left(non_empty_y, top) # inclusive - end = bisect_left(non_empty_y, bottom, begin) # exclusive - self.dirty.update(range(top, bottom)) + # if we were requested to clear the whole screen and + # the cursor's attrs are the same than the screen's default + # then this is equivalent to delete all the lines from the buffer + if (how == 2 or how == 3) and self.default_char == self.cursor.attrs: + buffer.clear() + return + # Remove the lines from the buffer as this is equivalent # to overwrite each char in them with the space character # (screen.default_char). # If a deleted line is then requested, a new line will # be added with screen.default_char as its default char if self.default_char == self.cursor.attrs: + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, top) # inclusive + end = bisect_left(non_empty_y, bottom, begin) # exclusive + for y in non_empty_y[begin:end]: del buffer[y] @@ -1274,16 +1281,10 @@ def erase_in_display(self, how=0, *args, **kwargs): data = self.cursor.attrs.data width = self.cursor.attrs.width style = self.cursor.attrs.style - for y in non_empty_y[begin:end]: + for y in range(top, bottom): line = buffer[y] write_data = line.write_data - # TODO: note that this may not be entirely correct as - # 'for x in line' iterates over the non-empty chars - # of the line, changing their data when write_data() - # is called. - # But this means that any empty char in the line - # is never touch, in particular, its attributes. - for x in line: + for x in range(0, self.columns): write_data(x, data, width, style) if how == 0 or how == 1: diff --git a/tests/test_screen.py b/tests/test_screen.py index 5af2e72..175e26c 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1977,6 +1977,43 @@ def test_erase_in_display(): ] consistency_asserts(screen) + screen.erase_in_display(3) + assert (screen.cursor.y, screen.cursor.x) == (2, 2) + assert screen.display == [" ", + " ", + " ", + " ", + " "] + assert tolist(screen) == [ + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + ] + consistency_asserts(screen) + + # erase a clean screen (reset) from the begin to cursor + screen.reset() + screen.cursor.y = 2 + screen.cursor.x = 2 + screen.select_graphic_rendition(31) # red foreground + + screen.erase_in_display(1) + assert (screen.cursor.y, screen.cursor.x) == (2, 2) + assert screen.display == [" ", + " ", + " ", + " ", + " "] + assert tolist(screen) == [ + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 5, + [Char(" ", fg="red")] * 3 + [screen.default_char] * 2, + [screen.default_char] * 5, + [screen.default_char] * 5, + ] + consistency_asserts(screen) def test_cursor_up(): screen = pyte.Screen(10, 10) From 47d7c62cb2a86436a3da97bdde845592f4bfc84a Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 20:47:09 -0300 Subject: [PATCH 38/54] Sparse iteration of insert_characters and delete_characters --- pyte/screens.py | 90 ++++++++++++++++++++++++++++++++-------- tests/helpers/asserts.py | 16 +++++++ tests/test_screen.py | 3 ++ 3 files changed, 91 insertions(+), 18 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index a542f3b..b7107e3 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1096,19 +1096,47 @@ def insert_characters(self, count=None): if not line: return - for x in range(self.columns, self.cursor.x - 1, -1): - new_x = x + count - if new_x <= self.columns: - if x in line: - line[x + count] = line.pop(x) - else: - # this is equivalent to: - # line[new_x] = line[x] - # where line[x] does not exist so line[new_x] - # should not exist either - line.pop(new_x, None) - else: - line.pop(x, None) + pop = line.pop + + # Note: the following is optimized for the case of long lines + # that are not very densely populated, the amount of count + # to insert is small and the cursor is not very close to the right + # end. + non_empty_x = sorted(line) + begin = bisect_left(non_empty_x, self.cursor.x) + end = bisect_left(non_empty_x, self.columns - count) + + to_move = reversed(non_empty_x[begin:end]) + + # cursor.x + # | + # V to_move + # |---------------| + # 0 1 x 3 4 5 count = 2 (x means empty) + # + # x x 0 1 4 3 (first for-loop without the inner loop: the "4" is wrong) + # + # x x 0 1 x 3 (first for-loop with the inner loop: the "4" is removed) + next_x = self.columns - count + for x in to_move: + # Notice how if (x + 1) == (next_x) then you know + # that no empty char are in between this x and the next one + # and therefore the range() loop gets empty. + # In other cases, (x + 1) < (next_x) + for z in range(x + 1 + count, next_x + count): + pop(z, None) + + # it may look weird but the current "x" is the "next_x" + # of the next iteration because we are iterating to_move + # backwards + next_x = x + line[x + count] = pop(x) + + # between the cursor.x and the last moved char + # we may have that should be emptied + for z in range(self.cursor.x, next_x + count): + pop(z, None) + def delete_characters(self, count=None): """Delete the indicated # of characters, starting with the @@ -1129,11 +1157,37 @@ def delete_characters(self, count=None): if not line: return - for x in range(self.cursor.x, self.columns): - if x + count <= self.columns: - line[x] = line.pop(x + count, self.default_char.copy()) - else: - line.pop(x, None) + pop = line.pop + + non_empty_x = sorted(line) + begin = bisect_left(non_empty_x, self.cursor.x + count) + + to_move = non_empty_x[begin:] + + # cursor.x + # | + # | to_move + # V |---------------| + # 0 1 x 3 4 x count = 2 (x means empty) + # + # 0 3 4 3 x x + # + # x 3 4 x x x + prev_x = self.cursor.x + count - 1 + for x in to_move: + # Notice how if (x - 1) == (prev_x) then you know + # that no empty char are in between this x and the prev one + # and therefore the range() loop gets empty. + # In other cases, (prev_x + 1) > (x) + for z in range(prev_x + 1 + count, x + count): + pop(z, None) + + prev_x = x + line[x - count] = pop(x) + + # this delete from the last written to the last read + for z in range(prev_x + 1 - count, prev_x + 1): + pop(z, None) def erase_characters(self, count=None): """Erase the indicated # of characters, starting with the diff --git a/tests/helpers/asserts.py b/tests/helpers/asserts.py index 17aa295..459d9ad 100644 --- a/tests/helpers/asserts.py +++ b/tests/helpers/asserts.py @@ -20,3 +20,19 @@ def consistency_asserts(screen): else: assert char.data == "" assert char.width == 0 + + # we check that no char is outside of the buffer + # we need to check the internal _buffer for this and do an educated + # check + non_empty_y = list(screen._buffer.keys()) + min_y = min(non_empty_y) if non_empty_y else 0 + max_y = max(non_empty_y) if non_empty_y else screen.lines - 1 + + assert 0 <= min_y <= max_y < screen.lines + + for line in screen._buffer.values(): + non_empty_x = list(line.keys()) + min_x = min(non_empty_x) if non_empty_x else 0 + max_x = max(non_empty_x) if non_empty_x else screen.columns - 1 + + assert 0 <= min_x <= max_x < screen.columns diff --git a/tests/test_screen.py b/tests/test_screen.py index 175e26c..b15dd80 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1367,6 +1367,7 @@ def test_insert_characters(): cursor = copy.copy(screen.cursor) screen.insert_characters(2) assert (screen.cursor.y, screen.cursor.x) == (cursor.y, cursor.x) + assert screen.display == [" s", "is ", "foo", "bar"] assert tolist(screen)[0] == [ screen.default_char, screen.default_char, @@ -1376,11 +1377,13 @@ def test_insert_characters(): # b) now inserting from the middle of the line screen.cursor.y, screen.cursor.x = 2, 1 screen.insert_characters(1) + assert screen.display == [" s", "is ", "f o", "bar"] assert tolist(screen)[2] == [Char("f"), screen.default_char, Char("o")] # c) inserting more than we have screen.cursor.y, screen.cursor.x = 3, 1 screen.insert_characters(10) + assert screen.display == [" s", "is ", "f o", "b "] assert tolist(screen)[3] == [ Char("b"), screen.default_char, screen.default_char ] From b4258e19d8fcc3c9a51f7eb79c453cb1f8fe549f Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 20:56:15 -0300 Subject: [PATCH 39/54] Add more checks to tests; fix bug on after_event and make it sparse-aware --- pyte/screens.py | 8 +++++--- tests/test_diff.py | 16 ++++++++++++++++ tests/test_screen.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index b7107e3..bbaebb5 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1715,11 +1715,13 @@ def after_event(self, event): :param str event: event name, for example ``"linefeed"``. """ if event in ["prev_page", "next_page"]: + columns = self.columns for line in self._buffer.values(): pop = line.pop - for x in line: - if x > self.columns: - pop(x) + non_empty_x = sorted(line.keys()) + begin = bisect_left(non_empty_x, columns) + for x in non_empty_x[begin:]: + pop(x) # If we're at the bottom of the history buffer and `DECTCEM` # mode is set -- show the cursor. diff --git a/tests/test_diff.py b/tests/test_diff.py index b8cc836..5ce16e8 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -1,6 +1,10 @@ import pyte from pyte import modes as mo +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) +from asserts import consistency_asserts + def test_mark_whole_screen(): # .. this is straightforward -- make sure we have a dirty attribute @@ -45,6 +49,7 @@ def test_mark_single_line(): getattr(screen, method)() assert len(screen.dirty) == 1 assert screen.cursor.y in screen.dirty + consistency_asserts(screen) def test_modes(): @@ -72,6 +77,7 @@ def test_index(): screen.cursor_to_line(24) screen.index() assert screen.dirty == set(range(screen.lines)) + consistency_asserts(screen) def test_reverse_index(): @@ -81,12 +87,14 @@ def test_reverse_index(): # a) not at the top margin -- whole screen is dirty. screen.reverse_index() assert screen.dirty == set(range(screen.lines)) + consistency_asserts(screen) # b) nothing is marked dirty. screen.dirty.clear() screen.cursor_to_line(screen.lines // 2) screen.reverse_index() assert not screen.dirty + consistency_asserts(screen) def test_insert_delete_lines(): @@ -97,6 +105,7 @@ def test_insert_delete_lines(): screen.dirty.clear() getattr(screen, method)() assert screen.dirty == set(range(screen.cursor.y, screen.lines)) + consistency_asserts(screen) def test_erase_in_display(): @@ -107,20 +116,24 @@ def test_erase_in_display(): screen.dirty.clear() screen.erase_in_display() assert screen.dirty == set(range(screen.cursor.y, screen.lines)) + consistency_asserts(screen) # b) from the beginning of the screen to cursor. screen.dirty.clear() screen.erase_in_display(1) assert screen.dirty == set(range(0, screen.cursor.y + 1)) + consistency_asserts(screen) # c) whole screen. screen.dirty.clear() screen.erase_in_display(2) assert screen.dirty == set(range(0, screen.lines)) + consistency_asserts(screen) screen.dirty.clear() screen.erase_in_display(3) assert screen.dirty == set(range(0, screen.lines)) + consistency_asserts(screen) def test_draw_wrap(): @@ -132,6 +145,7 @@ def test_draw_wrap(): screen.draw("g") assert screen.cursor.y == 0 screen.dirty.clear() + consistency_asserts(screen) # now write one more character which should cause wrapping screen.draw("h") @@ -139,6 +153,7 @@ def test_draw_wrap(): # regression test issue #36 where the wrong line was marked as # dirty assert screen.dirty == set([0, 1]) + consistency_asserts(screen) def test_draw_multiple_chars_wrap(): @@ -147,3 +162,4 @@ def test_draw_multiple_chars_wrap(): screen.draw("1234567890") assert screen.cursor.y == 1 assert screen.dirty == set([0, 1]) + consistency_asserts(screen) diff --git a/tests/test_screen.py b/tests/test_screen.py index b15dd80..1911fa4 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -231,6 +231,7 @@ def test_resize(): screen.set_margins(0, 1) assert screen.columns == screen.lines == 2 assert tolist(screen) == [[screen.default_char, screen.default_char]] * 2 + consistency_asserts(screen) screen.resize(3, 3) assert screen.columns == screen.lines == 3 @@ -239,10 +240,12 @@ def test_resize(): ] * 3 assert mo.DECOM in screen.mode assert screen.margins is None + consistency_asserts(screen) screen.resize(2, 2) assert screen.columns == screen.lines == 2 assert tolist(screen) == [[screen.default_char, screen.default_char]] * 2 + consistency_asserts(screen) # Quirks: # a) if the current display is narrower than the requested size, @@ -280,6 +283,7 @@ def test_resize_same(): screen.dirty.clear() screen.resize(2, 2) assert not screen.dirty + consistency_asserts(screen) def test_set_mode(): @@ -341,6 +345,7 @@ def test_draw(): # ... one` more character -- now we got a linefeed! screen.draw("a") assert (screen.cursor.y, screen.cursor.x) == (1, 1) + consistency_asserts(screen) # ``DECAWM`` is off. screen = pyte.Screen(3, 3) @@ -404,6 +409,7 @@ def test_draw_width2(): screen = pyte.Screen(10, 1) screen.draw("コンニチハ") assert screen.cursor.x == screen.columns + consistency_asserts(screen) def test_draw_width2_line_end(): @@ -411,6 +417,7 @@ def test_draw_width2_line_end(): screen = pyte.Screen(10, 1) screen.draw(" コンニチハ") assert screen.cursor.x == screen.columns + consistency_asserts(screen) @pytest.mark.xfail @@ -467,11 +474,13 @@ def test_draw_width0_decawm_off(): screen.reset_mode(mo.DECAWM) screen.draw(" コンニチハ") assert screen.cursor.x == screen.columns + consistency_asserts(screen) # The following should not advance the cursor. screen.draw("\N{ZERO WIDTH SPACE}") screen.draw("\u0007") # DELETE. assert screen.cursor.x == screen.columns + consistency_asserts(screen) def test_draw_cp437(): @@ -520,6 +529,7 @@ def test_carriage_return(): screen.carriage_return() assert screen.cursor.x == 0 + consistency_asserts(screen) def test_index(): @@ -536,6 +546,7 @@ def test_index(): [Char("w"), Char("o")], [Char("o", fg="red"), Char("t", fg="red")] ] + consistency_asserts(screen) # b) indexing on the last row should push everything up and # create a new row at the bottom. @@ -546,6 +557,7 @@ def test_index(): [Char("o", fg="red"), Char("t", fg="red")], [screen.default_char, screen.default_char] ] + consistency_asserts(screen) # c) same with margins screen = update(pyte.Screen(2, 5), ["bo", "sh", "th", "er", "oh"], @@ -645,6 +657,7 @@ def test_index_sparse(): [screen.default_char] * 5, [Char("x")] + [screen.default_char] * 3 + [Char("z")], ] + consistency_asserts(screen) # b) indexing on the last row should push everything up and # create a new row at the bottom. @@ -667,6 +680,7 @@ def test_index_sparse(): [Char("x")] + [screen.default_char] * 3 + [Char("z")], [screen.default_char] * 5, ] + consistency_asserts(screen) # again screen.index() @@ -685,6 +699,7 @@ def test_index_sparse(): [screen.default_char] * 5, [screen.default_char] * 5, ] + consistency_asserts(screen) # leave the screen cleared screen.index() @@ -705,6 +720,7 @@ def test_index_sparse(): [screen.default_char] * 5, [screen.default_char] * 5, ] + consistency_asserts(screen) def test_reverse_index(): @@ -718,6 +734,7 @@ def test_reverse_index(): [screen.default_char, screen.default_char], [Char("w", fg="red"), Char("o", fg="red")] ] + consistency_asserts(screen) # b) once again ... screen.reverse_index() @@ -726,6 +743,7 @@ def test_reverse_index(): [screen.default_char, screen.default_char], [screen.default_char, screen.default_char], ] + consistency_asserts(screen) # c) same with margins screen = update(pyte.Screen(2, 5), ["bo", "sh", "th", "er", "oh"], @@ -804,6 +822,7 @@ def test_reverse_index_sparse(): " ", "x z", ] + consistency_asserts(screen) # a) reverse indexing on the first row should push rows down # and create a new row at the top. @@ -823,6 +842,7 @@ def test_reverse_index_sparse(): [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], [screen.default_char] * 5, ] + consistency_asserts(screen) # again screen.reverse_index() @@ -841,6 +861,7 @@ def test_reverse_index_sparse(): [screen.default_char] * 5, [screen.default_char, Char("o", fg="red"), screen.default_char, Char("t", fg="red"), screen.default_char], ] + consistency_asserts(screen) # again screen.reverse_index() @@ -859,6 +880,7 @@ def test_reverse_index_sparse(): [Char("w"), Char("o"),] + [screen.default_char] * 3, [screen.default_char] * 5, ] + consistency_asserts(screen) # leave the screen cleared screen.reverse_index() @@ -878,6 +900,7 @@ def test_reverse_index_sparse(): [screen.default_char] * 5, [screen.default_char] * 5, ] + consistency_asserts(screen) def test_linefeed(): screen = update(pyte.Screen(2, 2), ["bo", "sh"], [None, None]) @@ -888,12 +911,14 @@ def test_linefeed(): screen.cursor.x, screen.cursor.y = 1, 0 screen.linefeed() assert (screen.cursor.y, screen.cursor.x) == (1, 0) + consistency_asserts(screen) # b) LNM off. screen.reset_mode(mo.LNM) screen.cursor.x, screen.cursor.y = 1, 0 screen.linefeed() assert (screen.cursor.y, screen.cursor.x) == (1, 1) + consistency_asserts(screen) def test_linefeed_margins(): @@ -902,8 +927,10 @@ def test_linefeed_margins(): screen.set_margins(3, 27) screen.cursor_position() assert (screen.cursor.y, screen.cursor.x) == (0, 0) + consistency_asserts(screen) screen.linefeed() assert (screen.cursor.y, screen.cursor.x) == (1, 0) + consistency_asserts(screen) def test_tabstops(): @@ -930,6 +957,7 @@ def test_tabstops(): assert screen.cursor.x == 9 screen.tab() assert screen.cursor.x == 9 + consistency_asserts(screen) def test_clear_tabstops(): @@ -944,6 +972,7 @@ def test_clear_tabstops(): screen.clear_tab_stop() assert screen.tabstops == set([1]) + consistency_asserts(screen) screen.set_tab_stop() screen.clear_tab_stop(0) @@ -957,6 +986,7 @@ def test_clear_tabstops(): screen.clear_tab_stop(3) assert not screen.tabstops + consistency_asserts(screen) def test_backspace(): @@ -967,6 +997,7 @@ def test_backspace(): screen.cursor.x = 1 screen.backspace() assert screen.cursor.x == 0 + consistency_asserts(screen) def test_save_cursor(): @@ -1008,6 +1039,7 @@ def test_save_cursor(): assert screen.cursor.attrs != screen.default_char assert screen.cursor.attrs == Char(" ", underscore=True) + consistency_asserts(screen) def test_restore_cursor_with_none_saved(): @@ -1031,6 +1063,7 @@ def test_restore_cursor_out_of_bounds(): screen.restore_cursor() assert (screen.cursor.y, screen.cursor.x) == (2, 2) + consistency_asserts(screen) # b) origin mode is on. screen.resize(10, 10) @@ -1043,6 +1076,7 @@ def test_restore_cursor_out_of_bounds(): screen.restore_cursor() assert (screen.cursor.y, screen.cursor.x) == (2, 4) + consistency_asserts(screen) def test_insert_lines(): @@ -1373,12 +1407,14 @@ def test_insert_characters(): screen.default_char, Char("s", fg="red") ] + consistency_asserts(screen) # b) now inserting from the middle of the line screen.cursor.y, screen.cursor.x = 2, 1 screen.insert_characters(1) assert screen.display == [" s", "is ", "f o", "bar"] assert tolist(screen)[2] == [Char("f"), screen.default_char, Char("o")] + consistency_asserts(screen) # c) inserting more than we have screen.cursor.y, screen.cursor.x = 3, 1 @@ -1389,6 +1425,7 @@ def test_insert_characters(): ] assert screen.display == [" s", "is ", "f o", "b "] + consistency_asserts(screen) # insert 1 at the begin of the previously edited line screen.cursor.y, screen.cursor.x = 3, 0 @@ -1396,6 +1433,7 @@ def test_insert_characters(): assert tolist(screen)[3] == [ screen.default_char, Char("b"), screen.default_char, ] + consistency_asserts(screen) # insert before the end of the line screen.cursor.y, screen.cursor.x = 3, 2 @@ -1403,6 +1441,7 @@ def test_insert_characters(): assert tolist(screen)[3] == [ screen.default_char, Char("b"), screen.default_char, ] + consistency_asserts(screen) # insert enough to push outside the screen the remaining char screen.cursor.y, screen.cursor.x = 3, 0 @@ -1412,6 +1451,7 @@ def test_insert_characters(): ] assert screen.display == [" s", "is ", "f o", " "] + consistency_asserts(screen) # d) 0 is 1 screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[0]) @@ -1422,6 +1462,7 @@ def test_insert_characters(): screen.default_char, Char("s", fg="red"), Char("a", fg="red") ] + consistency_asserts(screen) screen = update(pyte.Screen(3, 3), ["sam", "is ", "foo"], colored=[0]) screen.cursor_position() @@ -1430,6 +1471,7 @@ def test_insert_characters(): screen.default_char, Char("s", fg="red"), Char("a", fg="red") ] + consistency_asserts(screen) # ! extreme cases. @@ -2084,6 +2126,7 @@ def test_cursor_back_last_column(): screen.cursor_back(5) assert screen.cursor.x == (screen.columns - 1) - 5 + consistency_asserts(screen) def test_cursor_forward(): From f1787129a48330246f611fadd6188e75b095ffd7 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 22:23:58 -0300 Subject: [PATCH 40/54] Impl sparse iteration for insert_lines/delete_lines; fix insert_characters --- pyte/screens.py | 52 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index bbaebb5..a3941a5 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1048,10 +1048,28 @@ def insert_lines(self, count=None): # If cursor is outside scrolling margins it -- do nothin'. if top <= self.cursor.y <= bottom: self.dirty.update(range(self.cursor.y, self.lines)) - for y in range(bottom, self.cursor.y - 1, -1): - if y + count <= bottom and y in self._buffer: - self._buffer[y + count] = self._buffer[y] - self._buffer.pop(y, None) + + # the following algorithm is similar to the one found + # in insert_characters except that operates over + # the lines (y range) and not the chars (x range) + buffer = self._buffer + pop = buffer.pop + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, self.cursor.y) + end = bisect_left(non_empty_y, (bottom + 1) - count) + + to_move = reversed(non_empty_y[begin:end]) + + next_y = (bottom + 1) - count + for y in to_move: + for z in range(y + 1 + count, next_y + count): + pop(z, None) + + next_y = y + buffer[y + count] = pop(y) + + for z in range(self.cursor.y, next_y + count): + pop(z, None) self.carriage_return() @@ -1069,11 +1087,25 @@ def delete_lines(self, count=None): # If cursor is outside scrolling margins -- do nothin'. if top <= self.cursor.y <= bottom: self.dirty.update(range(self.cursor.y, self.lines)) - for y in range(self.cursor.y, bottom + 1): - if y + count <= bottom and y + count in self._buffer: - self._buffer[y] = self._buffer.pop(y + count) - else: - self._buffer.pop(y, None) + + buffer = self._buffer + pop = buffer.pop + non_empty_y = sorted(buffer) + begin = bisect_left(non_empty_y, self.cursor.y + count) + end = bisect_left(non_empty_y, bottom + 1) + + to_move = non_empty_y[begin:end] + + prev_y = self.cursor.y + count - 1 + for y in to_move: + for z in range(prev_y + 1 - count, y - count): + pop(z, None) + + prev_y = y + buffer[y - count] = pop(y) + + for z in range(prev_y + 1 - count, min(prev_y + 1, bottom + 1)): + pop(z, None) self.carriage_return() @@ -1179,7 +1211,7 @@ def delete_characters(self, count=None): # that no empty char are in between this x and the prev one # and therefore the range() loop gets empty. # In other cases, (prev_x + 1) > (x) - for z in range(prev_x + 1 + count, x + count): + for z in range(prev_x + 1 - count, x - count): pop(z, None) prev_x = x From 05c8c2c46453c4514be2d0ffa8ed63e149c605ee Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 22:32:00 -0300 Subject: [PATCH 41/54] Sparse iter for erase_characters/erase_in_line --- pyte/screens.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index a3941a5..685c520 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1244,9 +1244,12 @@ def erase_characters(self, count=None): # a char in the line is equivalent to delete it if from the line if line.default == self.cursor.attrs: pop = line.pop - for x in range(self.cursor.x, - min(self.cursor.x + count, self.columns)): - pop(x, None) + non_empty_x = sorted(line) + begin = bisect_left(non_empty_x, self.cursor.x) + end = bisect_left(non_empty_x, self.cursor.x + count, begin) + + for x in non_empty_x[begin:end]: + pop(x) # the line may end up being empty, delete it from the buffer (*) if not line: @@ -1280,11 +1283,11 @@ def erase_in_line(self, how=0, private=False): """ self.dirty.add(self.cursor.y) if how == 0: - interval = range(self.cursor.x, self.columns) + low, high = self.cursor.x, self.columns elif how == 1: - interval = range(self.cursor.x + 1) + low, high = 0, (self.cursor.x + 1) elif how == 2: - interval = range(self.columns) + low, high = 0, self.columns line = self._buffer[self.cursor.y] @@ -1292,8 +1295,12 @@ def erase_in_line(self, how=0, private=False): # a char in the line is equivalent to delete it if from the line if line.default == self.cursor.attrs: pop = line.pop - for x in interval: - pop(x, None) + non_empty_x = sorted(line) + begin = bisect_left(non_empty_x, low) + end = bisect_left(non_empty_x, high, begin) + + for x in non_empty_x[begin:end]: + pop(x) # the line may end up being empty, delete it from the buffer (*) if not line: @@ -1306,7 +1313,7 @@ def erase_in_line(self, how=0, private=False): style = self.cursor.attrs.style # a full range scan is required and not a sparse scan # because we were asked to *write* on that full range - for x in interval: + for x in range(low, high): write_data(x, data, width, style) def erase_in_display(self, how=0, *args, **kwargs): From 80aa50a2a00c5196f96e84f2427957038a5d249f Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Fri, 8 Jul 2022 22:38:41 -0300 Subject: [PATCH 42/54] Impl resize with sparse iter --- pyte/screens.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 685c520..9a552b2 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -622,8 +622,11 @@ def resize(self, lines=None, columns=None): if columns < self.columns: for line in self._buffer.values(): - for x in range(columns, self.columns): - line.pop(x, None) + pop = line.pop + non_empty_x = sorted(line) + begin = bisect_left(non_empty_x, columns) + for x in non_empty_x[begin:]: + pop(x) self.lines, self.columns = lines, columns self.set_margins() From 1bee165c6674406111e66d0b4cd1758bad399100 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 14:54:53 -0300 Subject: [PATCH 43/54] Optional use a dummy set for tracking dirty lines By default Screen tracks which lines are dirty and should be of interest for the user. This functionality may not be of interest for all the use cases and this tracking is expensibe, specially for large geometries. If track_dirty_lines is set to False, the screen.dirty attribute becomes a dummy or null set that it is always empty saving memory and time. --- benchmark.py | 7 ++++++- pyte/screens.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/benchmark.py b/benchmark.py index b52787e..0b928f7 100644 --- a/benchmark.py +++ b/benchmark.py @@ -40,7 +40,12 @@ def setup(path, screen_cls, columns, lines): with io.open(path, "rb") as handle: data = handle.read() - screen = screen_cls(columns, lines) + if screen_cls == pyte.Screen: + extra_args = {'track_dirty_lines': False} + else: + extra_args = {} + + screen = screen_cls(columns, lines, **extra_args) stream = pyte.ByteStream(screen) return data, screen, stream diff --git a/pyte/screens.py b/pyte/screens.py index 9a552b2..2d53bf3 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -25,6 +25,7 @@ :license: LGPL, see LICENSE for more details. """ +import collections import copy import json import math @@ -376,6 +377,26 @@ def __getitem__(self, y): def __len__(self): return self._screen.lines +class NullSet(collections.abc.MutableSet): + ''' Implementation of a set that it is always empty. ''' + def __contains__(self, x): + return False + + def __iter__(self): + return iter(set()) + + def __len__(self): + return 0 + + def add(self, x): + return + + def discard(self, x): + return + + def update(self, it): + return + class Screen: """ A screen is an in-memory matrix of characters that represents the @@ -455,12 +476,12 @@ def default_char(self): def default_line(self): return Line(self.default_char) - def __init__(self, columns, lines): + def __init__(self, columns, lines, track_dirty_lines=True): self.savepoints = [] self.columns = columns self.lines = lines self._buffer = defaultdict(lambda: Line(self.default_char)) - self.dirty = set() + self.dirty = set() if track_dirty_lines else NullSet() self._default_style = CharStyle( fg="default", bg="default", bold=False, @@ -1878,7 +1899,8 @@ def prev_page(self): # anyways (aka, we remove the old ones) pop(y, None) - self.dirty = set(range(self.lines)) + self.dirty.clear() + self.dirty.update(range(self.lines)) def next_page(self): """Move the screen page down through the history buffer.""" @@ -1936,7 +1958,8 @@ def next_page(self): # anyways (aka, we remove the old ones) pop(y, None) - self.dirty = set(range(self.lines)) + self.dirty.clear() + self.dirty.update(range(self.lines)) class DebugEvent(namedtuple("Event", "name args kwargs")): From 065b31d45ee9b03b115aa00cd70eb7bbfbf7cd27 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 15:08:36 -0300 Subject: [PATCH 44/54] Binary search for the tabstop --- pyte/screens.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 2d53bf3..0a5cb83 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1010,14 +1010,16 @@ def tab(self): """Move to the next tab space, or the end of the screen if there aren't anymore left. """ - for stop in sorted(self.tabstops): - if self.cursor.x < stop: - column = stop - break + tabstops = sorted(self.tabstops) + + # use bisect_right because self.cursor.x must not + # be included + at = bisect_right(tabstops, self.cursor.x) + if at == len(tabstops): + # no tabstops found, set the x to the end of the screen + self.cursor.x = self.columns - 1 else: - column = self.columns - 1 - - self.cursor.x = column + self.cursor.x = tabstops[at] def backspace(self): """Move cursor to the left one or keep it in its position if @@ -1425,7 +1427,7 @@ def clear_tab_stop(self, how=0): # present, or silently fails if otherwise. self.tabstops.discard(self.cursor.x) elif how == 3: - self.tabstops = set() # Clears all horizontal tab stops. + self.tabstops.clear() # Clears all horizontal tab stops. def ensure_hbounds(self): """Ensure the cursor is within horizontal screen bounds.""" From 8f110492abbee961296f9d84913caaab671b1c33 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 15:19:01 -0300 Subject: [PATCH 45/54] Make (0, lines-1) the default margin instead of None Instead of using None as a special case, set the margins to (0, lines-1) by default. This may break user code if the user is expecting None as a valid value or if it is setting it. If required we could make screen.margins a property and hide the internal implementation. --- pyte/screens.py | 30 +++++++++++++++--------------- tests/test_screen.py | 10 ++++++---- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 0a5cb83..50dde23 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -432,8 +432,8 @@ class Screen: (see :meth:`index` and :meth:`reverse_index`). Characters added outside the scrolling region do not make the screen to scroll. - The value is ``None`` if margins are set to screen boundaries, - otherwise -- a pair 0-based top and bottom line indices. + The margins are a pair 0-based top and bottom line indices + set to screen boundaries by default. .. attribute:: charset @@ -587,7 +587,7 @@ def reset(self): """ self.dirty.update(range(self.lines)) self._buffer.clear() - self.margins = None + self.margins = Margins(0, self.lines - 1) self.mode = set([mo.DECAWM, mo.DECTCEM]) @@ -660,10 +660,10 @@ def set_margins(self, top=None, bottom=None): """ # XXX 0 corresponds to the CSI with no parameters. if (top is None or top == 0) and bottom is None: - self.margins = None + self.margins = Margins(0, self.lines - 1) return - margins = self.margins or Margins(0, self.lines - 1) + margins = self.margins # Arguments are 1-based, while :attr:`margins` are zero # based -- so we have to decrement them by one. We also @@ -940,7 +940,7 @@ def index(self): """Move the cursor down one line in the same column. If the cursor is at the last line, create a new line at the bottom. """ - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins if self.cursor.y == bottom: buffer = self._buffer pop = buffer.pop @@ -970,7 +970,7 @@ def reverse_index(self): """Move the cursor up one line in the same column. If the cursor is at the first line, create a new line at the top. """ - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins if self.cursor.y == top: buffer = self._buffer pop = buffer.pop @@ -1069,7 +1069,7 @@ def insert_lines(self, count=None): :param count: number of lines to insert. """ count = count or 1 - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins # If cursor is outside scrolling margins it -- do nothin'. if top <= self.cursor.y <= bottom: @@ -1108,7 +1108,7 @@ def delete_lines(self, count=None): :param int count: number of lines to delete. """ count = count or 1 - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins # If cursor is outside scrolling margins -- do nothin'. if top <= self.cursor.y <= bottom: @@ -1441,7 +1441,7 @@ def ensure_vbounds(self, use_margins=None): cursor is bounded by top and and bottom margins, instead of ``[0; lines - 1]``. """ - if (use_margins or mo.DECOM in self.mode) and self.margins is not None: + if (use_margins or mo.DECOM in self.mode): top, bottom = self.margins else: top, bottom = 0, self.lines - 1 @@ -1454,7 +1454,7 @@ def cursor_up(self, count=None): :param int count: number of lines to skip. """ - top, _bottom = self.margins or Margins(0, self.lines - 1) + top, _bottom = self.margins self.cursor.y = max(self.cursor.y - (count or 1), top) def cursor_up1(self, count=None): @@ -1472,7 +1472,7 @@ def cursor_down(self, count=None): :param int count: number of lines to skip. """ - _top, bottom = self.margins or Margins(0, self.lines - 1) + _top, bottom = self.margins self.cursor.y = min(self.cursor.y + (count or 1), bottom) def cursor_down1(self, count=None): @@ -1522,7 +1522,7 @@ def cursor_position(self, line=None, column=None): # If origin mode (DECOM) is set, line number are relative to # the top scrolling margin. - if self.margins is not None and mo.DECOM in self.mode: + if mo.DECOM in self.mode: line += self.margins.top # Cursor is not allowed to move out of the scrolling region. @@ -1817,7 +1817,7 @@ def erase_in_display(self, how=0, *args, **kwargs): def index(self): """Overloaded to update top history with the removed lines.""" - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins if self.cursor.y == bottom: self.history.top.append(self.buffer[top]) @@ -1826,7 +1826,7 @@ def index(self): def reverse_index(self): """Overloaded to update bottom history with the removed lines.""" - top, bottom = self.margins or Margins(0, self.lines - 1) + top, bottom = self.margins if self.cursor.y == top: self.history.bottom.append(self.buffer[bottom]) diff --git a/tests/test_screen.py b/tests/test_screen.py index 1911fa4..803df02 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -227,8 +227,10 @@ def test_attributes_reset(): def test_resize(): screen = pyte.Screen(2, 2) + assert screen.margins == (0, 1) screen.set_mode(mo.DECOM) screen.set_margins(0, 1) + assert screen.margins == (0, 1) assert screen.columns == screen.lines == 2 assert tolist(screen) == [[screen.default_char, screen.default_char]] * 2 consistency_asserts(screen) @@ -239,7 +241,7 @@ def test_resize(): [screen.default_char, screen.default_char, screen.default_char] ] * 3 assert mo.DECOM in screen.mode - assert screen.margins is None + assert screen.margins == (0, 2) consistency_asserts(screen) screen.resize(2, 2) @@ -2218,7 +2220,7 @@ def test_alignment_display(): def test_set_margins(): screen = pyte.Screen(10, 10) - assert screen.margins is None + assert screen.margins == (0, 9) # a) ok-case screen.set_margins(1, 5) @@ -2231,7 +2233,7 @@ def test_set_margins(): # c) no margins provided -- reset to full screen. screen.set_margins() - assert screen.margins is None + assert screen.margins == (0, 9) def test_set_margins_zero(): @@ -2240,7 +2242,7 @@ def test_set_margins_zero(): screen.set_margins(1, 5) assert screen.margins == (0, 4) screen.set_margins(0) - assert screen.margins is None + assert screen.margins == (0, 23) def test_hide_cursor(): From 01b7d56bd3c5ee881a7712e52e33ede840fd541a Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 15:39:44 -0300 Subject: [PATCH 46/54] Make internal _buffer a dict and not a defaultdict Using a defaultdict can easily introduce false entries in the buffer making it less sparse. The new Buffer class supports all the dict's operations but it does not add an entry if the key is missing. To add a new entry (a new line), do a buffer.line_at(y). This is equivalent to dict.setdefault(y, new_line()) but avoids the call to new_line() if an entry y exists. --- pyte/screens.py | 29 ++++++++++++++++++++--------- tests/test_screen.py | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 50dde23..3f061f4 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -33,7 +33,7 @@ import sys import unicodedata import warnings -from collections import deque, namedtuple, defaultdict +from collections import deque, namedtuple from functools import lru_cache from bisect import bisect_left, bisect_right @@ -317,6 +317,17 @@ def stats(self, screen): span=(max(self) - min(self)) if self else None ) +class Buffer(dict): + __slots__ = ('_screen', ) + def __init__(self, screen): + self._screen = screen + + def line_at(self, y): + try: + return self[y] + except KeyError: + self[y] = line = self._screen.default_line() + return line class LineView: """ @@ -480,7 +491,7 @@ def __init__(self, columns, lines, track_dirty_lines=True): self.savepoints = [] self.columns = columns self.lines = lines - self._buffer = defaultdict(lambda: Line(self.default_char)) + self._buffer = Buffer(self) self.dirty = set() if track_dirty_lines else NullSet() self._default_style = CharStyle( @@ -826,7 +837,7 @@ def draw(self, data): # so we fetch them here and update accordingly if necessary cursor_x = cursor.x cursor_y = cursor.y - line = buffer[cursor_y] + line = buffer.line_at(cursor_y) write_data = line.write_data char_at = line.char_at @@ -852,7 +863,7 @@ def draw(self, data): # linefeed may update cursor.y so we update cursor_y and # the current line accordingly. cursor_y = cursor.y - line = buffer[cursor_y] + line = buffer.line_at(cursor_y) write_data = line.write_data char_at = line.char_at elif char_width > 0: @@ -894,7 +905,7 @@ def draw(self, data): normalized = unicodedata.normalize("NFC", last.data + char) last.data = normalized elif cursor_y: - last = buffer[cursor_y - 1].char_at(columns - 1) + last = buffer.line_at(cursor_y - 1).char_at(columns - 1) normalized = unicodedata.normalize("NFC", last.data + char) last.data = normalized else: @@ -1264,7 +1275,7 @@ def erase_characters(self, count=None): self.dirty.add(self.cursor.y) count = count or 1 - line = self._buffer[self.cursor.y] + line = self._buffer.line_at(self.cursor.y) # If the line's default char is equivalent to our cursor, overwriting # a char in the line is equivalent to delete it if from the line @@ -1315,7 +1326,7 @@ def erase_in_line(self, how=0, private=False): elif how == 2: low, high = 0, self.columns - line = self._buffer[self.cursor.y] + line = self._buffer.line_at(self.cursor.y) # If the line's default char is equivalent to our cursor, overwriting # a char in the line is equivalent to delete it if from the line @@ -1401,7 +1412,7 @@ def erase_in_display(self, how=0, *args, **kwargs): width = self.cursor.attrs.width style = self.cursor.attrs.style for y in range(top, bottom): - line = buffer[y] + line = buffer.line_at(y) write_data = line.write_data for x in range(0, self.columns): write_data(x, data, width, style) @@ -1569,7 +1580,7 @@ def alignment_display(self): self.dirty.update(range(self.lines)) style = self._default_style for y in range(self.lines): - line = self._buffer[y] + line = self._buffer.line_at(y) for x in range(self.columns): line.write_data(x, "E", wcwidth("E"), style) diff --git a/tests/test_screen.py b/tests/test_screen.py index 803df02..96ea85c 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -39,7 +39,7 @@ def update(screen, lines, colored=[], write_spaces=True): # skip, leave the default char in the screen pass else: - screen._buffer[y].write_data(x, char, 1, style) + screen._buffer.line_at(y).write_data(x, char, 1, style) return screen From e3fdf41deae5e3e287e44867ddf5458beab68429 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 20:00:07 -0300 Subject: [PATCH 47/54] Optionally disable display graphic attributes When the graphic attributes are disabled, select_graphic_rendition always set the default style. With this, the lines' default char and the cursor's char will always match and the screen will optimize the erase* methods. --- pyte/screens.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 3f061f4..054c468 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -487,12 +487,13 @@ def default_char(self): def default_line(self): return Line(self.default_char) - def __init__(self, columns, lines, track_dirty_lines=True): + def __init__(self, columns, lines, track_dirty_lines=True, disable_display_graphic=False): self.savepoints = [] self.columns = columns self.lines = lines self._buffer = Buffer(self) self.dirty = set() if track_dirty_lines else NullSet() + self.disabled_display_graphic = disable_display_graphic self._default_style = CharStyle( fg="default", bg="default", bold=False, @@ -1592,7 +1593,7 @@ def select_graphic_rendition(self, *attrs): replace = {} # Fast path for resetting all attributes. - if not attrs or attrs == (0, ): + if not attrs or attrs == (0, ) or self.disabled_display_graphic: self.cursor.attrs = self.default_char return else: From 7839ded9065bbf5b3ca8c79dec7a727e275a87e0 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 20:03:36 -0300 Subject: [PATCH 48/54] Replace explicit for-loops with map calls --- pyte/screens.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 054c468..651a547 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -658,8 +658,8 @@ def resize(self, lines=None, columns=None): pop = line.pop non_empty_x = sorted(line) begin = bisect_left(non_empty_x, columns) - for x in non_empty_x[begin:]: - pop(x) + + list(map(pop, non_empty_x[begin:])) self.lines, self.columns = lines, columns self.set_margins() @@ -1286,8 +1286,7 @@ def erase_characters(self, count=None): begin = bisect_left(non_empty_x, self.cursor.x) end = bisect_left(non_empty_x, self.cursor.x + count, begin) - for x in non_empty_x[begin:end]: - pop(x) + list(map(pop, non_empty_x[begin:end])) # the line may end up being empty, delete it from the buffer (*) if not line: @@ -1337,8 +1336,7 @@ def erase_in_line(self, how=0, private=False): begin = bisect_left(non_empty_x, low) end = bisect_left(non_empty_x, high, begin) - for x in non_empty_x[begin:end]: - pop(x) + list(map(pop, non_empty_x[begin:end])) # the line may end up being empty, delete it from the buffer (*) if not line: @@ -1401,12 +1399,12 @@ def erase_in_display(self, how=0, *args, **kwargs): # If a deleted line is then requested, a new line will # be added with screen.default_char as its default char if self.default_char == self.cursor.attrs: + pop = buffer.pop non_empty_y = sorted(buffer) begin = bisect_left(non_empty_y, top) # inclusive end = bisect_left(non_empty_y, bottom, begin) # exclusive - for y in non_empty_y[begin:end]: - del buffer[y] + list(map(pop, non_empty_y[begin:end])) else: data = self.cursor.attrs.data @@ -1795,10 +1793,10 @@ def after_event(self, event): columns = self.columns for line in self._buffer.values(): pop = line.pop - non_empty_x = sorted(line.keys()) + non_empty_x = sorted(line) begin = bisect_left(non_empty_x, columns) - for x in non_empty_x[begin:]: - pop(x) + + list(map(pop, non_empty_x[begin:])) # If we're at the bottom of the history buffer and `DECTCEM` # mode is set -- show the cursor. From 2ca29a51c60b2e176f6bbd19987ff507d78df6ac Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sat, 9 Jul 2022 21:33:22 -0300 Subject: [PATCH 49/54] Pass track_dirty_lines and disable_display_graphic to HistoryScreen --- benchmark.py | 38 +++++++++++++++++++++----------------- pyte/screens.py | 7 +++++-- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/benchmark.py b/benchmark.py index 0b928f7..b5bfc7d 100644 --- a/benchmark.py +++ b/benchmark.py @@ -36,59 +36,63 @@ import pyte -def setup(path, screen_cls, columns, lines): +def setup(path, screen_cls, columns, lines, optimize_conf): with io.open(path, "rb") as handle: data = handle.read() - if screen_cls == pyte.Screen: - extra_args = {'track_dirty_lines': False} - else: - extra_args = {} + extra_args = {} + if optimize_conf: + extra_args = { + 'track_dirty_lines': False, + 'disable_display_graphic': True, + } screen = screen_cls(columns, lines, **extra_args) stream = pyte.ByteStream(screen) return data, screen, stream -def make_stream_feed_benchmark(path, screen_cls, columns, lines): - data, _, stream = setup(path, screen_cls, columns, lines) +def make_stream_feed_benchmark(path, screen_cls, columns, lines, optimize_conf): + data, _, stream = setup(path, screen_cls, columns, lines, optimize_conf) return partial(stream.feed, data) -def make_screen_display_benchmark(path, screen_cls, columns, lines): - data, screen, stream = setup(path, screen_cls, columns, lines) +def make_screen_display_benchmark(path, screen_cls, columns, lines, optimize_conf): + data, screen, stream = setup(path, screen_cls, columns, lines, optimize_conf) stream.feed(data) return lambda: screen.display -def make_screen_reset_benchmark(path, screen_cls, columns, lines): - data, screen, stream = setup(path, screen_cls, columns, lines) +def make_screen_reset_benchmark(path, screen_cls, columns, lines, optimize_conf): + data, screen, stream = setup(path, screen_cls, columns, lines, optimize_conf) stream.feed(data) return screen.reset -def make_screen_resize_half_benchmark(path, screen_cls, columns, lines): - data, screen, stream = setup(path, screen_cls, columns, lines) +def make_screen_resize_half_benchmark(path, screen_cls, columns, lines, optimize_conf): + data, screen, stream = setup(path, screen_cls, columns, lines, optimize_conf) stream.feed(data) return partial(screen.resize, lines=lines//2, columns=columns//2) if __name__ == "__main__": benchmark = os.environ["BENCHMARK"] lines, columns = map(int, os.environ.get("GEOMETRY", "24x80").split('x')) - sys.argv.extend(["--inherit-environ", "BENCHMARK,GEOMETRY"]) + optimize_conf = int(os.environ.get("OPTIMIZECONF", "0")) + sys.argv.extend(["--inherit-environ", "BENCHMARK,GEOMETRY,OPTIMIZECONF"]) runner = Runner() metadata = { 'input_file': benchmark, 'columns': columns, - 'lines': lines + 'lines': lines, + 'optimize_conf': optimize_conf } benchmark_name = os.path.basename(benchmark) - for screen_cls in [pyte.Screen, pyte.DiffScreen, pyte.HistoryScreen]: + for screen_cls in [pyte.Screen, pyte.HistoryScreen]: screen_cls_name = screen_cls.__name__ for make_test in (make_stream_feed_benchmark, make_screen_display_benchmark, make_screen_reset_benchmark, make_screen_resize_half_benchmark): scenario = make_test.__name__[5:-10] # remove make_ and _benchmark name = f"[{scenario} {lines}x{columns}] {benchmark_name}->{screen_cls_name}" metadata.update({'scenario': scenario, 'screen_cls': screen_cls_name}) - runner.bench_func(name, make_test(benchmark, screen_cls, columns, lines), metadata=metadata) + runner.bench_func(name, make_test(benchmark, screen_cls, columns, lines, optimize_conf), metadata=metadata) diff --git a/pyte/screens.py b/pyte/screens.py index 651a547..4644b84 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1748,14 +1748,17 @@ class HistoryScreen(Screen): _wrapped = set(Stream.events) _wrapped.update(["next_page", "prev_page"]) - def __init__(self, columns, lines, history=100, ratio=.5): + def __init__(self, columns, lines, history=100, ratio=.5, + track_dirty_lines=True, disable_display_graphic=False): self.history = History(deque(maxlen=history), deque(maxlen=history), float(ratio), history, history) - super(HistoryScreen, self).__init__(columns, lines) + super(HistoryScreen, self).__init__(columns, lines, + track_dirty_lines=track_dirty_lines, + disable_display_graphic=disable_display_graphic) def _make_wrapper(self, event, handler): def inner(*args, **kwargs): From c5892658a70f758e3f8e6e60ced97cb1471883f7 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 10 Jul 2022 12:10:24 -0300 Subject: [PATCH 50/54] Document what are and how to interpret the LineStats and BufferStats --- pyte/screens.py | 90 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 4644b84..30dbf8e 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -77,12 +77,46 @@ class LineStats(namedtuple("_LineStats", [ "empty", "chars", "columns", - "occupacy", + "occupancy", "min", "max", "span", ])): """ + LineStats contains some useful statistics about a single line in + the screen to understand how the terminal program draw on it + and how pyte.Screen makes use of the line. + + The basic statistic is the character count over the total count + of columns of the screen. The line is implemented as a sparse + buffer so space characters are not really stored and this ratio + reflects how many non-space chars are. The ratio is also known + as occupancy. + + A ratio close to 0 means that the line is mostly empty + and it will consume little memory and the screen's algorithms + will run faster; close to 1 means that it is mostly full and it will + have the opposite effects. + + For non-empty lines, the second statistic useful is its range. + The range of a line is the minimum and maximum x coordinates + where we find a non-space char. + For a screen of 80 columns, a range of [10 - 20] means that + the chars up to the x=10 are spaces and the chars after x=20 + are spaces too. + The length of the range, also known as the span, is calculated. + The chars/span ratio gives you how densely packed is the range. + + A ratio close to 0 means that the chars are sparse within the range + so it is too fragmented and the screen's algorithms will have to jump + between the chars and the gaps having a lower performance. + A ratio close to 1 means that they are highly packed and the screen + will have a better performance. + + With the ratios char/columns and chars/span one can understand + the balance of sparsity, its distribution and how it will impact + on the memory and execution time. + Note: this is not part of the stable API so it may change between version of pyte. """ @@ -90,12 +124,12 @@ class LineStats(namedtuple("_LineStats", [ def __repr__(self): if self.empty: return "chars: {0: >3}/{1} ({2:.2f})".format( - self.chars, self.columns, self.occupacy, + self.chars, self.columns, self.occupancy, ) else: return "chars: {0: >3}/{1} ({2:.2f}); range: [{3: >3} - {4: >3}], len: {5: >3} ({6:.2f})".format( - self.chars, self.columns, self.occupacy, - self.min, self.max, self.span, self.span/self.columns + self.chars, self.columns, self.occupancy, + self.min, self.max, self.span, self.chars/self.span ) class BufferStats(namedtuple("_BufferStats", [ @@ -105,13 +139,49 @@ class BufferStats(namedtuple("_BufferStats", [ "lines", "falses", "blanks", - "occupacy", + "occupancy", "min", "max", "span", "line_stats", ])): """ + BufferStats has some statistics about the buffer of the screen, + a 2d sparse matrix representation of the screen. + + The sparse implementation means that empty lines are not stored + in the buffer explicitly. + + The stats count the real lines (aka entries) and the ratio entries + over total lines that the screen has (aka occupancy). + + A ratio close to 0 means that the buffer is mostly empty + and it will consume little memory and the screen's algorithms + will run faster; close to 1 means that it is mostly full and it will + have the opposite effects. + + The buffer may have entries for empty lines in two forms: + + - falses lines: empty lines that are the same as the buffer's default + and therefore should not be in the buffer at all + - empty lines: non-empty lines but full of spaces. It is suspicious + because a line full of spaces should not have entries within but + there are legit cases when this is not true: for example when + the terminal program erase some chars typing space chars with + a non-default cursor attributes. + + Both counts are part of the stats with their falses/entries + and blanks/entries ratios. + + For non-empty buffers the minimum and maximum y-coordinates + are part of the stats. From there, the range and the span (length) + are calculated as well the entries/span ratio to see how densely + packed are the lines. + See LineStats for more about these stats. + + After the buffer's stats, the stats of each non-empty line in the buffer + follows. See LineStats for that. + Note: this is not part of the stable API so it may change between version of pyte. """ @@ -126,7 +196,7 @@ def __repr__(self): if self.empty: return bstats + \ "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}), blanks: {5:> 3} ({6:.2f})\n{7}".format( - self.entries, self.lines, self.occupacy, + self.entries, self.lines, self.occupancy, self.falses, self.falses/self.entries, self.blanks, self.blanks/self.entries, "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) @@ -134,10 +204,10 @@ def __repr__(self): else: return bstats + \ "line entries: {0: >3}/{1} ({2:.2f}), falses: {3:> 3} ({4:.2f}), blanks: {5:> 3} ({6:.2f}); range: [{7: >3} - {8: >3}], len: {9: >3} ({10:.2f})\n{11}".format( - self.entries, self.lines, self.occupacy, + self.entries, self.lines, self.occupancy, self.falses, self.falses/self.entries, self.blanks, self.blanks/self.entries, - self.min, self.max, self.span, self.span/self.lines, + self.min, self.max, self.span, self.entries/self.span, "\n".join("{0: >3}: {1}".format(x, stats) for x, stats in self.line_stats) ) @@ -311,7 +381,7 @@ def stats(self, screen): empty=not bool(self), chars=len(self), columns=screen.columns, - occupacy=len(self)/screen.columns, + occupancy=len(self)/screen.columns, min=min(self) if self else None, max=max(self) if self else None, span=(max(self) - min(self)) if self else None @@ -524,7 +594,7 @@ def stats(self): lines=self.lines, falses=len([line for line in buffer.values() if not line]), blanks=len([line for line in buffer.values() if all(char.data == " " for char in line.values())]), - occupacy=len(buffer)/self.lines, + occupancy=len(buffer)/self.lines, min=min(buffer) if buffer else None, max=max(buffer) if buffer else None, span=(max(buffer) - min(buffer)) if buffer else None, From 1b42c89c63be0c73491da87bea257a0e4af95fa4 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 10 Jul 2022 14:20:29 -0300 Subject: [PATCH 51/54] Fuzzy tests *_characters and *_lines methods --- tests/helpers/asserts.py | 42 ++++++++ tests/test_screen.py | 213 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 253 insertions(+), 2 deletions(-) diff --git a/tests/helpers/asserts.py b/tests/helpers/asserts.py index 459d9ad..f359480 100644 --- a/tests/helpers/asserts.py +++ b/tests/helpers/asserts.py @@ -36,3 +36,45 @@ def consistency_asserts(screen): max_x = max(non_empty_x) if non_empty_x else screen.columns - 1 assert 0 <= min_x <= max_x < screen.columns + + +def splice(seq, at, count, padding, margins=None): + ''' Take a sequence and add count padding objects at the + given position "at". + If count is negative, instead of adding, remove + objects at the given position and append the same + amount at the end. + + If margins=(low, high) are given, operate between + the low and the high indexes of the sequence. + These are 0-based indexes, both inclusive. + ''' + + assert count != 0 + assert isinstance(seq, list) + + low, high = margins if margins else (0, len(seq) - 1) + + if not (low <= at <= high): + return list(seq) + + low_part = seq[:low] + high_part = seq[high+1:] + + middle = seq[low:high+1] + at = at - low # "at" now is an index of middle, not of seq. + + if count < 0: # remove mode + count = abs(count) + l = len(middle) + del middle[at:at+count] + middle += padding * (l - len(middle)) + else: # insert mode + middle = middle[:at] + padding * count + middle[at:] + del middle[-count:] + + new = low_part + middle + high_part + assert len(new) == len(seq) + return new + + diff --git a/tests/test_screen.py b/tests/test_screen.py index 96ea85c..9b2032e 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -1,4 +1,4 @@ -import copy, sys, os +import copy, sys, os, itertools import pytest @@ -7,7 +7,7 @@ from pyte.screens import Char as _orig_Char, CharStyle sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) -from asserts import consistency_asserts +from asserts import consistency_asserts, splice # Implement the old API of Char so we don't have to change # all the tests @@ -2313,3 +2313,212 @@ def test_screen_set_icon_name_title(): screen.set_title(text) assert screen.title == text + + +def test_fuzzy_insert_characters(): + columns = 7 + + # test different one-line screen scenarios with a mix + # of empty and non-empty chars + for mask in itertools.product('x ', repeat=columns): + # make each 'x' a different letter so we can spot subtle errors + line = [c if m == 'x' else ' ' for m, c in zip(mask, 'ABCDEFGHIJK')] + assert len(line) == columns + original = list(line) + for count in [1, 2, columns//2, columns-1, columns, columns+1]: + for at in [0, 1, columns//2, columns-count, columns-count+1, columns-1]: + if at < 0: + continue + + for margins in [None, (1, columns-2)]: + screen = update(pyte.Screen(columns, 1), [line], write_spaces=False) + + if margins: + # set_margins is 1-based indexes + screen.set_margins(top=margins[0]+1, bottom=margins[1]+1) + + screen.cursor.x = at + screen.insert_characters(count) + + # screen.insert_characters are not margins-aware so they + # will ignore any margin set. Therefore the expected + # line should also ignore them + expected_line = splice(line, at, count, [" "], margins=None) + expected_line = ''.join(expected_line) + + assert screen.display == [expected_line], "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + consistency_asserts(screen) + + # map the chars to Char objects + expected_line = [screen.default_char if c == ' ' else Char(c) for c in expected_line] + assert tolist(screen)[0] == expected_line, "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + + # ensure that the line that we used for the tests was not modified + # so the tests used the correct line object (otherwise the tests + # are invalid) + assert original == line + + + +def test_fuzzy_delete_characters(): + columns = 7 + + # test different one-line screen scenarios with a mix + # of empty and non-empty chars + for mask in itertools.product('x ', repeat=columns): + line = [c if m == 'x' else ' ' for m, c in zip(mask, 'ABCDEFGHIJK')] + assert len(line) == columns + original = list(line) + for count in [1, 2, columns//2, columns-1, columns, columns+1]: + for at in [0, 1, columns//2, columns-count, columns-count+1, columns-1]: + if at < 0: + continue + for margins in [None, (1, columns-2)]: + screen = update(pyte.Screen(columns, 1), [line], write_spaces=False) + + if margins: + # set_margins is 1-based indexes + screen.set_margins(top=margins[0]+1, bottom=margins[1]+1) + + screen.cursor.x = at + screen.delete_characters(count) + + # screen.delete_characters are not margins-aware so they + # will ignore any margin set. Therefore the expected + # line should also ignore them + expected_line = splice(line, at, (-1)*count, [" "], margins=None) + expected_line = ''.join(expected_line) + + assert screen.display == [expected_line], "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + consistency_asserts(screen) + + # map the chars to Char objects + expected_line = [screen.default_char if c == ' ' else Char(c) for c in expected_line] + assert tolist(screen)[0] == expected_line, "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + + # ensure that the line that we used for the tests was not modified + # so the tests used the correct line object (otherwise the tests + # are invalid) + assert original == line + + + + +def test_fuzzy_erase_characters(): + columns = 7 + + # test different one-line screen scenarios with a mix + # of empty and non-empty chars + for mask in itertools.product('x ', repeat=columns): + line = [c if m == 'x' else ' ' for m, c in zip(mask, 'ABCDEFGHIJK')] + assert len(line) == columns + original = list(line) + for count in [1, 2, columns//2, columns-1, columns, columns+1]: + for at in [0, 1, columns//2, columns-count, columns-count+1, columns-1]: + if at < 0: + continue + for margins in [None, (1, columns-2)]: + screen = update(pyte.Screen(columns, 1), [line], write_spaces=False) + + if margins: + # set_margins is 1-based indexes + screen.set_margins(top=margins[0]+1, bottom=margins[1]+1) + + screen.cursor.x = at + screen.erase_characters(count) + + expected_line = list(line) + expected_line[at:at+count] = [" "] * (min(at+count, columns) - at) + expected_line = ''.join(expected_line) + + assert screen.display == [expected_line], "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + consistency_asserts(screen) + + # map the chars to Char objects + expected_line = [screen.default_char if c == ' ' else Char(c) for c in expected_line] + assert tolist(screen)[0] == expected_line, "At {}, cnt {}, (m {}), initial line: {}".format(at, count, margins, line) + + # ensure that the line that we used for the tests was not modified + # so the tests used the correct line object (otherwise the tests + # are invalid) + assert original == line + + +def test_fuzzy_insert_lines(): + rows = 7 + + # test different screen scenarios with a mix + # of empty and non-empty lines + for masks in itertools.product(['x x', ' '], repeat=rows): + # make each line different + lines = [m if m == ' ' else '%c %c' % (c,c) for m, c in zip(masks, "ABCDEFGHIJK")] + assert len(lines) == rows + original = list(lines) + for count in [1, 2, rows//2, rows-1, rows, rows+1]: + for at in [0, 1, rows//2, rows-count, rows-count+1, rows-1]: + if at < 0: + continue + for margins in [None, (1, rows-2)]: + screen = update(pyte.Screen(3, rows), lines, write_spaces=False) + + if margins: + # set_margins is 1-based indexes + screen.set_margins(top=margins[0]+1, bottom=margins[1]+1) + + screen.cursor.y = at + screen.insert_lines(count) + + expected_lines = splice(lines, at, count, [" "], margins) + + assert screen.display == expected_lines, "At {}, cnt {}, (m {}), initial lines: {}".format(at, count, margins, lines) + consistency_asserts(screen) + + # map the chars to Char objects + expected_lines = [[screen.default_char if c == ' ' else Char(c) for c in l] for l in expected_lines] + assert tolist(screen) == expected_lines, "At {}, cnt {}, (m {}), initial lines: {}".format(at, count, margins, lines) + + # ensure that the line that we used for the tests was not modified + # so the tests used the correct line object (otherwise the tests + # are invalid) + assert original == lines + + + +def test_fuzzy_delete_lines(): + rows = 7 + + # test different screen scenarios with a mix + # of empty and non-empty lines + for masks in itertools.product(['x x', ' '], repeat=rows): + lines = [m if m == ' ' else '%c %c' % (c,c) for m, c in zip(masks, "ABCDEFGHIJK")] + assert len(lines) == rows + original = list(lines) + for count in [1, 2, rows//2, rows-1, rows, rows+1]: + for at in [0, 1, rows//2, rows-count, rows-count+1, rows-1]: + if at < 0: + continue + for margins in [None, (1, rows-2)]: + screen = update(pyte.Screen(3, rows), lines, write_spaces=False) + + if margins: + # set_margins is 1-based indexes + screen.set_margins(top=margins[0]+1, bottom=margins[1]+1) + + screen.cursor.y = at + screen.delete_lines(count) + + expected_lines = splice(lines, at, (-1)*count, [" "], margins) + + assert screen.display == expected_lines, "At {}, cnt {}, (m {}), initial lines: {}".format(at, count, margins, lines) + consistency_asserts(screen) + + # map the chars to Char objects + expected_lines = [[screen.default_char if c == ' ' else Char(c) for c in l] for l in expected_lines] + assert tolist(screen) == expected_lines, "At {}, cnt {}, (m {}), initial lines: {}".format(at, count, margins, lines) + + # ensure that the line that we used for the tests was not modified + # so the tests used the correct line object (otherwise the tests + # are invalid) + assert original == lines + + From c65ac82167037ce61fe3fa80e84b5e0b73214b85 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 10 Jul 2022 15:42:32 -0300 Subject: [PATCH 52/54] Improve the docs (plus minor fixes on Line and BufferView) --- AUTHORS | 1 + pyte/screens.py | 175 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 144 insertions(+), 32 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5b2f5a5..5934494 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,3 +17,4 @@ Contributors - Byron Roosa - Andrew Crozier - @eight04 +- Martin Di Paola @eldipa diff --git a/pyte/screens.py b/pyte/screens.py index 30dbf8e..2a91297 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -20,7 +20,7 @@ how to do -- feel free to submit a pull request. :copyright: (c) 2011-2012 by Selectel. - :copyright: (c) 2012-2017 by pyte authors and contributors, + :copyright: (c) 2012-2022 by pyte authors and contributors, see AUTHORS for details. :license: LGPL, see LICENSE for more details. """ @@ -83,9 +83,9 @@ class LineStats(namedtuple("_LineStats", [ "span", ])): """ - LineStats contains some useful statistics about a single line in - the screen to understand how the terminal program draw on it - and how pyte.Screen makes use of the line. + :class:`~pyte.screens.LineStats` contains some useful statistics + about a single line in the screen to understand how the terminal program + draw on it and how :class:`~pyte.screens.Screen` makes use of the line. The basic statistic is the character count over the total count of columns of the screen. The line is implemented as a sparse @@ -117,7 +117,9 @@ class LineStats(namedtuple("_LineStats", [ the balance of sparsity, its distribution and how it will impact on the memory and execution time. - Note: this is not part of the stable API so it may change + .. note:: + + This is not part of the stable API so it may change between version of pyte. """ @@ -146,8 +148,8 @@ class BufferStats(namedtuple("_BufferStats", [ "line_stats", ])): """ - BufferStats has some statistics about the buffer of the screen, - a 2d sparse matrix representation of the screen. + :class:`~pyte.screens.BufferStats` has some statistics about + the buffer of the screen, a 2d sparse matrix representation of the screen. The sparse implementation means that empty lines are not stored in the buffer explicitly. @@ -177,12 +179,14 @@ class BufferStats(namedtuple("_BufferStats", [ are part of the stats. From there, the range and the span (length) are calculated as well the entries/span ratio to see how densely packed are the lines. - See LineStats for more about these stats. + See :class:`~pyte.screens.LineStats` for more about these stats. After the buffer's stats, the stats of each non-empty line in the buffer - follows. See LineStats for that. + follows. See :class:`~pyte.screens.LineStats` for that. + + .. note:: - Note: this is not part of the stable API so it may change + This is not part of the stable API so it may change between version of pyte. """ @@ -213,9 +217,21 @@ def __repr__(self): class Char: - """A single styled on-screen character. + """ + A single styled on-screen character. The character is made + of an unicode character (data), its width and its style. :param str data: unicode character. Invariant: ``len(data) == 1``. + :param bool width: the width in terms of cells to display this char. + :param CharStyle style: the style of the character. + + The :meth:`~pyte.screens.Char.from_attributes` allows to create + a new :class:`~pyte.screens.Char` object + setting each attribute, one by one, without requiring and explicit + :class:`~pyte.screens.CharStyle` object. + + The supported attributes are: + :param str fg: foreground colour. Defaults to ``"default"``. :param str bg: background colour. Defaults to ``"default"``. :param bool bold: flag for rendering the character using bold font. @@ -230,7 +246,11 @@ class Char: during rendering. Defaults to ``False``. :param bool blink: flag for rendering the character blinked. Defaults to ``False``. - :param bool width: the width in terms of cells to display this char. + + The attributes data, width and style of :class:`~pyte.screens.Char` + must be considered read-only. Any modification is undefined. + If you want to modify a :class:`~pyte.screens.Char`, use the public + interface of :class:`~pyte.screens.Screen`. """ __slots__ = ( "data", @@ -347,6 +367,15 @@ def __init__(self, x, y, attrs): class Line(dict): + """A line or row of the screen. + + This dict subclass implements a sparse array for 0-based + indexed characters that represents a single line or row of the screen. + + :param pyte.screens.Char default: a :class:`~pyte.screens.Char` instance + to be used as default. See :meth:`~pyte.screens.Line.char_at` + for details. + """ __slots__ = ('default', ) def __init__(self, default): self.default = default @@ -354,7 +383,8 @@ def __init__(self, default): def write_data(self, x, data, width, style): """ Update the char at the position x with the new data, width and style. - If no char is at that position, a new char is created. + If no char is at that position, a new char is created and added + to the line. """ if x in self: char = self[x] @@ -365,16 +395,27 @@ def write_data(self, x, data, width, style): self[x] = Char(data, width, style) def char_at(self, x): - if x in self: + """ + Return the character at the given position x. If no char exists, + create a new one and add it to the line before returning it. + + This is a shortcut of `line.setdefault(x, line.default.copy())` + but avoids the copy if the char already exists. + """ + try: return self[x] - else: - char = self.default.copy() - self[x] = char + except KeyError: + self[x] = char = self.default.copy() return char def stats(self, screen): """ - Note: this is not part of the stable API so it may change + Return a :class:`~pyte.screens.LineStats` object with the statistics + of the line. + + .. note:: + + This is not part of the stable API so it may change between version of pyte. """ return LineStats( @@ -388,11 +429,29 @@ def stats(self, screen): ) class Buffer(dict): + """A 2d matrix representation of the screen. + + This dict subclass implements a sparse array for 0-based + indexed lines that represents the screen. Each line is then + a sparse array for the characters in the same row (see + :class:`~pyte.screens.Line`). + + :param pyte.screens.Screen screen: a :class:`~pyte.screens.Screen` instance + to be used when a default line needs to be created. + See :meth:`~pyte.screens.Buffer.line_at` for details. + """ __slots__ = ('_screen', ) def __init__(self, screen): self._screen = screen def line_at(self, y): + """ + Return the line at the given position y. If no line exists, + create a new one and add it to the buffer before returning it. + + This is a shortcut of `buffer.setdefault(y, screen.default_line())` + but avoids the copy if the line already exists. + """ try: return self[y] except KeyError: @@ -403,11 +462,13 @@ class LineView: """ A read-only view of an horizontal line of the screen. + :param pyte.screens.Line line: a :class:`~pyte.screens.Line` instance + Modifications to the internals of the screen is still possible through - this LineView however any modification will result in an undefined - behaviour. Don't do that. + this :class:`~pyte.screens.LineView` however any modification + will result in an undefined behaviour. Don't do that. - See BufferView. + See :class:`~pyte.screens.BufferView`. """ __slots__ = ("_line",) def __init__(self, line): @@ -436,12 +497,26 @@ class BufferView: """ A read-only view of the screen. + :param pyte.screens.Screen screen: a :class:`~pyte.screens.Screen` instance + Modifications to the internals of the screen is still possible through - this BufferView however any modification will result in an undefined - behaviour. Don't do that. + this :class:`~pyte.screens.BufferView` however any modification + will result in an undefined behaviour. Don't do that. - Any modification to the screen must be done through its method - (principally draw()) + Any modification to the screen must be done through its methods + (principally :meth:`~pyte.screens.Screen.draw`). + + This view allows the user to iterate over the lines and chars of + the buffer to query their attributes. + + As an example: + + view = screen.buffer # get a BufferView + for y in view: + line = view[y] # get a LineView (do it once per y line) + for x in line: + char = line[x] # get a Char + print(char.data, char.fg, char.bg) # access to char's attrs """ __slots__ = ("_buffer", "_screen") def __init__(self, screen): @@ -449,8 +524,9 @@ def __init__(self, screen): self._buffer = screen._buffer def __getitem__(self, y): - line = self._buffer.get(y) - if line is None: + try: + line = self._buffer[y] + except KeyError: line = Line(self._screen.default_char) return LineView(line) @@ -458,8 +534,8 @@ def __getitem__(self, y): def __len__(self): return self._screen.lines -class NullSet(collections.abc.MutableSet): - ''' Implementation of a set that it is always empty. ''' +class _NullSet(collections.abc.MutableSet): + """Implementation of a set that it is always empty.""" def __contains__(self, x): return False @@ -485,9 +561,30 @@ class Screen: and given explicit commands, or it can be attached to a stream and will respond to events. + :param int columns: count of columns for the screen (width). + :param int lines: count of lines for the screen (height). + + :param bool track_dirty_lines: track which lines were modified + (see `dirty` attribute). If it is false do not track any line. + Defaults to True. + + :param bool disable_display_graphic: disables the modification + of cursor attributes disabling :meth:`~pyte.screens.Screen.select_graphic_rendition`. + Defaults to False. + + .. note:: + + If you don't need the functionality, setting `track_dirty_lines` + to False and `disable_display_graphic` to True can + make :class:`~pyte.screens.Screen` to work faster and consume less + resources. + .. attribute:: buffer - A sparse ``lines x columns`` :class:`~pyte.screens.Char` matrix. + A ``lines x columns`` :class:`~pyte.screens.Char` matrix view of + the screen. Under the hood :class:`~pyte.screens.Screen` implements + a sparse matrix but `screen.buffer` returns a dense view. + See :class:`~pyte.screens.BufferView` .. attribute:: dirty @@ -500,6 +597,9 @@ class Screen: >>> list(screen.dirty) [0] + If `track_dirty_lines` was set to false, this `dirty` set will be + always empty. + .. versionadded:: 0.7.0 .. attribute:: cursor @@ -562,7 +662,7 @@ def __init__(self, columns, lines, track_dirty_lines=True, disable_display_graph self.columns = columns self.lines = lines self._buffer = Buffer(self) - self.dirty = set() if track_dirty_lines else NullSet() + self.dirty = set() if track_dirty_lines else _NullSet() self.disabled_display_graphic = disable_display_graphic self._default_style = CharStyle( @@ -583,7 +683,11 @@ def buffer(self): def stats(self): """ - Note: this is not part of the stable API so it may change + Return the statistcs of the buffer. + + .. note:: + + This is not part of the stable API so it may change between version of pyte. """ buffer = self._buffer @@ -1657,6 +1761,13 @@ def select_graphic_rendition(self, *attrs): """Set display attributes. :param list attrs: a list of display attributes to set. + + .. note:: + + If `disable_display_graphic` was set, this method + set the cursor's attributes to the default char's attributes + ignoring all the parameters. + Equivalent to `screen.select_graphic_rendition(0)`. """ replace = {} From f4ea46a76d32e623f4916a0d6ae5a5483da6bf30 Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Sun, 10 Jul 2022 16:16:36 -0300 Subject: [PATCH 53/54] Impl compressed_display to optionally avoid left/right spaces, top/bottom empty lines This is an optimization over screen.display where it is possible to strip left/right spaces of the lines and/or filter top/bottom whole empty lines. It is implemented in an opportunistic fashion so it may not fully strip/filter all that it is expected. In particular, lines with left or right spaces with non-default attributes are not stripped; lines that contains only spaces at the top or bottom are not filtered (even if lstrip/rstrip is set). This implementation is meant to be used when the screen has very large geometries and screen.display wastes too much time and memory on padding large lines and/or filling with a lot of empty lines. --- pyte/screens.py | 58 ++++++++++++++++++++++++-------- tests/test_screen.py | 80 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 14 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 2a91297..02bcf86 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -705,30 +705,55 @@ def stats(self): line_stats=[(x, line.stats(self)) for x, line in sorted(buffer.items())] ) - @property - def display(self): - """A :func:`list` of screen lines as unicode strings.""" + def compressed_display(self, lstrip=False, rstrip=False, tfilter=False, bfilter=False): + """A :func:`list` of screen lines as unicode strings with optionally + the possibility to compress its output striping space and filtering + empty lines. + + :param bool lstrip: strip the left space of each line. + :param bool rstrip: strip the right space of each line. + :param bool tfilter: filter the top whole empty lines. + :param bool bfilter: filter the bottom whole empty lines. + + .. note:: + + The strip of left/right spaces on each line and/or the filter + of top/bottom whole empty lines is implemented in an opportunistic + fashion and it may not strip/filter fully the spaces and/or lines. + + This method is meant to be an optimization over + :meth:`~pyte.screens.Screen.display` for displaying + large mostly-empty screens. + + For left-written texts, + `compressed_display(rstrip=True, tfilter=True, bfilter=True)` compress + the display without losing meaning. + + For right-written texts, + `compressed_display(lstrip=True, tfilter=True, bfilter=True)` compress + the display without losing meaning. + """ # screen.default_char is always the space character # We can skip the lookup of it and set the padding char # directly + empty_line_padding = "" if (lstrip or rstrip) else " " padding = " " - prev_y = -1 + non_empty_y = sorted(self._buffer.items()) + prev_y = non_empty_y[0][0]-1 if tfilter and non_empty_y else -1 output = [] columns = self.columns - for y, line in sorted(self._buffer.items()): + for y, line in non_empty_y: empty_lines = y - (prev_y + 1) if empty_lines: - output.extend([padding * columns] * empty_lines) + output.extend([empty_line_padding * columns] * empty_lines) prev_y = y + non_empty_x = sorted(line.items()) is_wide_char = False - prev_x = -1 + prev_x = non_empty_x[0][0]-1 if lstrip and non_empty_x else -1 display_line = [] - for x, cell in sorted(line.items()): - if x >= columns: - break - + for x, cell in non_empty_x: gap = x - (prev_x + 1) if gap: display_line.append(padding * gap) @@ -743,17 +768,22 @@ def display(self): display_line.append(char) gap = columns - (prev_x + 1) - if gap: + if gap and not rstrip: display_line.append(padding * gap) output.append("".join(display_line)) empty_lines = self.lines - (prev_y + 1) - if empty_lines: - output.extend([padding * columns] * empty_lines) + if empty_lines and not bfilter: + output.extend([empty_line_padding * columns] * empty_lines) return output + @property + def display(self): + """A :func:`list` of screen lines as unicode strings.""" + return self.compressed_display() + def reset(self): """Reset the terminal to its initial state. diff --git a/tests/test_screen.py b/tests/test_screen.py index 9b2032e..ed49883 100644 --- a/tests/test_screen.py +++ b/tests/test_screen.py @@ -2522,3 +2522,83 @@ def test_fuzzy_delete_lines(): assert original == lines +def test_compressed_display(): + screen = update(pyte.Screen(4, 5), [ + " ", + " a ", + " ", + " bb", + " ", + ], write_spaces=False) + + assert screen.display == [ + " ", + " a ", + " ", + " bb", + " ", + ] + + assert screen.compressed_display() == [ + " ", + " a ", + " ", + " bb", + " ", + ] + + assert screen.compressed_display(lstrip=True) == [ + "", + "a ", + "", + "bb", + "", + ] + + assert screen.compressed_display(rstrip=True) == [ + "", + " a", + "", + " bb", + "", + ] + + assert screen.compressed_display(lstrip=True, rstrip=True) == [ + "", + "a", + "", + "bb", + "", + ] + + assert screen.compressed_display(tfilter=True) == [ + " a ", + " ", + " bb", + " ", + ] + + assert screen.compressed_display(bfilter=True) == [ + " ", + " a ", + " ", + " bb", + ] + + assert screen.compressed_display(tfilter=True, bfilter=True) == [ + " a ", + " ", + " bb", + ] + + assert screen.compressed_display(tfilter=True, bfilter=True, rstrip=True) == [ + " a", + "", + " bb", + ] + + assert screen.compressed_display(tfilter=True, bfilter=True, lstrip=True) == [ + "a ", + "", + "bb", + ] From ba980a08ebb0b077bcce6cacd667ea325e639f0d Mon Sep 17 00:00:00 2001 From: Martin Di Paola Date: Tue, 12 Jul 2022 18:02:06 -0300 Subject: [PATCH 54/54] Optimize the insert/delete characters/lines with map-loops --- pyte/screens.py | 110 +++++++++++++++--------------------------------- 1 file changed, 34 insertions(+), 76 deletions(-) diff --git a/pyte/screens.py b/pyte/screens.py index 02bcf86..636c138 100644 --- a/pyte/screens.py +++ b/pyte/screens.py @@ -1297,21 +1297,14 @@ def insert_lines(self, count=None): buffer = self._buffer pop = buffer.pop non_empty_y = sorted(buffer) - begin = bisect_left(non_empty_y, self.cursor.y) - end = bisect_left(non_empty_y, (bottom + 1) - count) + move_begin = bisect_left(non_empty_y, self.cursor.y) + drop_begin = bisect_left(non_empty_y, (bottom + 1) - count, move_begin) + margin_begin = bisect_left(non_empty_y, bottom + 1, drop_begin) - to_move = reversed(non_empty_y[begin:end]) + list(map(pop, non_empty_y[drop_begin:margin_begin])) # drop - next_y = (bottom + 1) - count - for y in to_move: - for z in range(y + 1 + count, next_y + count): - pop(z, None) - - next_y = y - buffer[y + count] = pop(y) - - for z in range(self.cursor.y, next_y + count): - pop(z, None) + for y in reversed(non_empty_y[move_begin:drop_begin]): + buffer[y + count] = pop(y) # move self.carriage_return() @@ -1333,21 +1326,14 @@ def delete_lines(self, count=None): buffer = self._buffer pop = buffer.pop non_empty_y = sorted(buffer) - begin = bisect_left(non_empty_y, self.cursor.y + count) - end = bisect_left(non_empty_y, bottom + 1) + drop_begin = bisect_left(non_empty_y, self.cursor.y) + margin_begin = bisect_left(non_empty_y, bottom + 1, drop_begin) + move_begin = bisect_left(non_empty_y, self.cursor.y + count, drop_begin, margin_begin) - to_move = non_empty_y[begin:end] - - prev_y = self.cursor.y + count - 1 - for y in to_move: - for z in range(prev_y + 1 - count, y - count): - pop(z, None) - - prev_y = y - buffer[y - count] = pop(y) + list(map(pop, non_empty_y[drop_begin:move_begin])) # drop - for z in range(prev_y + 1 - count, min(prev_y + 1, bottom + 1)): - pop(z, None) + for y in non_empty_y[move_begin:margin_begin]: + buffer[y - count] = pop(y) # move self.carriage_return() @@ -1377,39 +1363,25 @@ def insert_characters(self, count=None): # to insert is small and the cursor is not very close to the right # end. non_empty_x = sorted(line) - begin = bisect_left(non_empty_x, self.cursor.x) - end = bisect_left(non_empty_x, self.columns - count) - - to_move = reversed(non_empty_x[begin:end]) + move_begin = bisect_left(non_empty_x, self.cursor.x) + drop_begin = bisect_left(non_empty_x, self.columns - count, move_begin) # cursor.x # | - # V to_move - # |---------------| + # V to_move to_drop + # |---------------|-------| # 0 1 x 3 4 5 count = 2 (x means empty) # - # x x 0 1 4 3 (first for-loop without the inner loop: the "4" is wrong) - # - # x x 0 1 x 3 (first for-loop with the inner loop: the "4" is removed) - next_x = self.columns - count - for x in to_move: - # Notice how if (x + 1) == (next_x) then you know - # that no empty char are in between this x and the next one - # and therefore the range() loop gets empty. - # In other cases, (x + 1) < (next_x) - for z in range(x + 1 + count, next_x + count): - pop(z, None) + list(map(pop, non_empty_x[drop_begin:])) # drop - # it may look weird but the current "x" is the "next_x" - # of the next iteration because we are iterating to_move - # backwards - next_x = x - line[x + count] = pop(x) + # cursor.x + # | + # V moved + # |---------------| + # x x 0 1 x 3 count = 2 (x means empty) + for x in reversed(non_empty_x[move_begin:drop_begin]): + line[x + count] = pop(x) # move - # between the cursor.x and the last moved char - # we may have that should be emptied - for z in range(self.cursor.x, next_x + count): - pop(z, None) def delete_characters(self, count=None): @@ -1434,34 +1406,20 @@ def delete_characters(self, count=None): pop = line.pop non_empty_x = sorted(line) - begin = bisect_left(non_empty_x, self.cursor.x + count) + drop_begin = bisect_left(non_empty_x, self.cursor.x) + move_begin = bisect_left(non_empty_x, self.cursor.x + count, drop_begin) - to_move = non_empty_x[begin:] + list(map(pop, non_empty_x[drop_begin:move_begin])) # drop # cursor.x - # | - # | to_move - # V |---------------| + # | + # V to drop to_move + # |-------|---------------| # 0 1 x 3 4 x count = 2 (x means empty) - # - # 0 3 4 3 x x - # - # x 3 4 x x x - prev_x = self.cursor.x + count - 1 - for x in to_move: - # Notice how if (x - 1) == (prev_x) then you know - # that no empty char are in between this x and the prev one - # and therefore the range() loop gets empty. - # In other cases, (prev_x + 1) > (x) - for z in range(prev_x + 1 - count, x - count): - pop(z, None) - - prev_x = x - line[x - count] = pop(x) - - # this delete from the last written to the last read - for z in range(prev_x + 1 - count, prev_x + 1): - pop(z, None) + # x x x 3 4 x after the drop + # x 3 4 x x x after the move + for x in non_empty_x[move_begin:]: + line[x - count] = pop(x) # move def erase_characters(self, count=None): """Erase the indicated # of characters, starting with the