From 7b52adf9bd8ce87bf880a72ee67031258a4d993d Mon Sep 17 00:00:00 2001 From: KV Date: Sun, 23 Jun 2024 21:34:18 +0200 Subject: [PATCH 1/2] Avoid ResourceWarning: unclosed file (#395) A number of such warnings showed up when running e.g. PYTHONWARNINGS=always python build_examples.py PYTHONWARNINGS=always wireviz ../../examples/demo0?.yml See https://github.com/wireviz/WireViz/pull/309#issuecomment-2170988381 Fix: All open() calls should be in a "with open() as x" statement to ensure closing the file when exiting the block in any way. Otherwise, use the new file_read_text() or file_write_text() functions to read or write the whole utf-8 text file and closing it. --- src/wireviz/Harness.py | 5 ++--- src/wireviz/wireviz.py | 4 ++-- src/wireviz/wv_cli.py | 6 +++--- src/wireviz/wv_helper.py | 21 +++++++++++++++++---- src/wireviz/wv_html.py | 10 +++++----- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/wireviz/Harness.py b/src/wireviz/Harness.py index b01049b7b..165e9fb0e 100644 --- a/src/wireviz/Harness.py +++ b/src/wireviz/Harness.py @@ -44,11 +44,10 @@ ) from wireviz.wv_helper import ( awg_equiv, + file_write_text, flatten2d, is_arrow, mm2_equiv, - open_file_read, - open_file_write, tuplelist2tsv, ) from wireviz.wv_html import generate_html_output @@ -693,7 +692,7 @@ def output( # BOM output bomlist = bom_list(self.bom()) if "tsv" in fmt: - open_file_write(f"{filename}.bom.tsv").write(tuplelist2tsv(bomlist)) + file_write_text(f"{filename}.bom.tsv", tuplelist2tsv(bomlist)) if "csv" in fmt: # TODO: implement CSV output (preferrably using CSV library) print("CSV output is not yet supported") diff --git a/src/wireviz/wireviz.py b/src/wireviz/wireviz.py index 00da16a50..18bd98483 100755 --- a/src/wireviz/wireviz.py +++ b/src/wireviz/wireviz.py @@ -15,9 +15,9 @@ from wireviz.Harness import Harness from wireviz.wv_helper import ( expand, + file_read_text, get_single_key_and_value, is_arrow, - open_file_read, smart_file_resolve, ) @@ -409,7 +409,7 @@ def _get_yaml_data_and_path(inp: Union[str, Path, Dict]) -> (Dict, Path): try: yaml_path = Path(inp).expanduser().resolve(strict=True) # if no FileNotFoundError exception happens, get file contents - yaml_str = open_file_read(yaml_path).read() + yaml_str = file_read_text(yaml_path) except (FileNotFoundError, OSError, ValueError) as e: # if inp is a long YAML string, Pathlib will normally raise # FileNotFoundError or OSError(errno = ENAMETOOLONG) when diff --git a/src/wireviz/wv_cli.py b/src/wireviz/wv_cli.py index afb024947..c83e1ccdf 100644 --- a/src/wireviz/wv_cli.py +++ b/src/wireviz/wv_cli.py @@ -11,7 +11,7 @@ import wireviz.wireviz as wv from wireviz import APP_NAME, __version__ -from wireviz.wv_helper import open_file_read +from wireviz.wv_helper import file_read_text format_codes = { # "c": "csv", @@ -111,7 +111,7 @@ def wireviz(file, format, prepend, output_dir, output_name, version): raise Exception(f"File does not exist:\n{prepend_file}") print("Prepend file:", prepend_file) - prepend_input += open_file_read(prepend_file).read() + "\n" + prepend_input += file_read_text(prepend_file) + "\n" else: prepend_input = "" @@ -130,7 +130,7 @@ def wireviz(file, format, prepend, output_dir, output_name, version): "Output file: ", f"{Path(_output_dir / _output_name)}.{output_formats_str}" ) - yaml_input = open_file_read(file).read() + yaml_input = file_read_text(file) file_dir = file.parent yaml_input = prepend_input + yaml_input diff --git a/src/wireviz/wv_helper.py b/src/wireviz/wv_helper.py index a10f6acf2..89fb9215e 100644 --- a/src/wireviz/wv_helper.py +++ b/src/wireviz/wv_helper.py @@ -113,18 +113,31 @@ def clean_whitespace(inp): def open_file_read(filename): + """Open utf-8 encoded text file for reading - remember closing it when finished""" # TODO: Intelligently determine encoding return open(filename, "r", encoding="UTF-8") def open_file_write(filename): + """Open utf-8 encoded text file for writing - remember closing it when finished""" return open(filename, "w", encoding="UTF-8") def open_file_append(filename): + """Open utf-8 encoded text file for appending - remember closing it when finished""" return open(filename, "a", encoding="UTF-8") +def file_read_text(filename: str) -> str: + """Read utf-8 encoded text file, close it, and return the text""" + return Path(filename).read_text(encoding="utf-8") + + +def file_write_text(filename: str, text: str) -> int: + """Write utf-8 encoded text file, close it, and return the number of characters written""" + return Path(filename).write_text(text, encoding="utf-8") + + def is_arrow(inp): """ Matches strings of one or multiple `-` or `=` (but not mixed) @@ -144,10 +157,10 @@ def aspect_ratio(image_src): try: from PIL import Image - image = Image.open(image_src) - if image.width > 0 and image.height > 0: - return image.width / image.height - print(f"aspect_ratio(): Invalid image size {image.width} x {image.height}") + with Image.open(image_src) as image: + if image.width > 0 and image.height > 0: + return image.width / image.height + print(f"aspect_ratio(): Invalid image size {image.width} x {image.height}") # ModuleNotFoundError and FileNotFoundError are the most expected, but all are handled equally. except Exception as error: print(f"aspect_ratio(): {type(error).__name__}: {error}") diff --git a/src/wireviz/wv_html.py b/src/wireviz/wv_html.py index 4052fdcf3..446f8c385 100644 --- a/src/wireviz/wv_html.py +++ b/src/wireviz/wv_html.py @@ -9,9 +9,9 @@ from wireviz.svgembed import data_URI_base64 from wireviz.wv_gv_html import html_line_breaks from wireviz.wv_helper import ( + file_read_text, + file_write_text, flatten2d, - open_file_read, - open_file_write, smart_file_resolve, ) @@ -35,14 +35,14 @@ def generate_html_output( # fall back to built-in simple template if no template was provided templatefile = Path(__file__).parent / "templates/simple.html" - html = open_file_read(templatefile).read() + html = file_read_text(templatefile) # embed SVG diagram (only if used) def svgdata() -> str: return re.sub( "^<[?]xml [^?>]*[?]>[^<]*]*>", "", - open_file_read(f"{filename}.tmp.svg").read(), + file_read_text(f"{filename}.tmp.svg"), 1, ) @@ -128,4 +128,4 @@ def replacement_if_used(key: str, func: Callable[[], str]) -> None: pattern = re.compile("|".join(replacements_escaped)) html = pattern.sub(lambda match: replacements[match.group(0)], html) - open_file_write(f"{filename}.html").write(html) + file_write_text(f"{filename}.html", html) From 969cd994999ec3578d123d7d031b154cd3e50177 Mon Sep 17 00:00:00 2001 From: KV Date: Fri, 5 Jul 2024 23:26:24 +0200 Subject: [PATCH 2/2] Add TODOs about utf-8 encoding/charset (#395) --- src/wireviz/Harness.py | 4 ++-- src/wireviz/svgembed.py | 4 ++-- src/wireviz/wv_html.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wireviz/Harness.py b/src/wireviz/Harness.py index 165e9fb0e..b823a9d63 100644 --- a/src/wireviz/Harness.py +++ b/src/wireviz/Harness.py @@ -171,7 +171,7 @@ def create_graph(self) -> Graph: bgcolor=wv_colors.translate_color(self.options.bgcolor, "HEX"), nodesep="0.33", fontname=self.options.fontname, - ) + ) # TODO: Add graph attribute: charset="utf-8", dot.attr( "node", shape="none", @@ -657,7 +657,7 @@ def png(self): return data.read() @property - def svg(self): + def svg(self): # TODO?: Verify xml encoding="utf-8" in SVG? graph = self.graph return embed_svg_images(graph.pipe(format="svg").decode("utf-8"), Path.cwd()) diff --git a/src/wireviz/svgembed.py b/src/wireviz/svgembed.py index c8d637c89..535fa4a95 100644 --- a/src/wireviz/svgembed.py +++ b/src/wireviz/svgembed.py @@ -59,8 +59,8 @@ def embed_svg_images_file( ) -> None: filename_in = Path(filename_in).resolve() filename_out = filename_in.with_suffix(".b64.svg") - filename_out.write_text( + filename_out.write_text( # TODO?: Verify xml encoding="utf-8" in SVG? embed_svg_images(filename_in.read_text(), filename_in.parent) - ) + ) # TODO: Use encoding="utf-8" in both read_text() and write_text() if overwrite: filename_out.replace(filename_in) diff --git a/src/wireviz/wv_html.py b/src/wireviz/wv_html.py index 446f8c385..19b77793f 100644 --- a/src/wireviz/wv_html.py +++ b/src/wireviz/wv_html.py @@ -35,11 +35,11 @@ def generate_html_output( # fall back to built-in simple template if no template was provided templatefile = Path(__file__).parent / "templates/simple.html" - html = file_read_text(templatefile) + html = file_read_text(templatefile) # TODO?: Warn if unexpected meta charset? # embed SVG diagram (only if used) def svgdata() -> str: - return re.sub( + return re.sub( # TODO?: Verify xml encoding="utf-8" in SVG? "^<[?]xml [^?>]*[?]>[^<]*]*>", "", file_read_text(f"{filename}.tmp.svg"),