From 21cfd917420624617a64ebee969565d9d77f6b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Kocha=C5=84czyk?= Date: Mon, 20 May 2024 18:03:56 +0200 Subject: [PATCH] Abrogate internal compression Also, fix illegible images as well. --- README.md | 8 ++------ maestro.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6d96056..238ef0e 100644 --- a/README.md +++ b/README.md @@ -214,12 +214,6 @@ Maestro may be used to get a glimpse of the exported data: Archives ========= -### Optional preliminary decompression -TIFF files exported by Harmony are LZW-compressed internally. To enable efficient external compression, it is strongly advised to avert this internal compression, e.g., with ImageMagick: -``` - find . -name '*.tiff' -exec mogrify -compress none {} \; -``` - ### Archiving One may call: @@ -228,6 +222,8 @@ One may call: ``` to obtain BZ2-compressed archives of images, one archive per well. +By default, internal compression is abrogated (TIFF files exported by Harmony are internally compressed with LZW, which prevents a more efficient external compression). + Note on compression ratio: bzip2, although slow, was experimentally checked to give the best compression ratio, exceeding that of zip and even xz at its "ultra" settings and extra-large dictionary, and is considered more suitable for long-term data storage than xz. diff --git a/maestro.py b/maestro.py index 9feccbe..d59c48a 100755 --- a/maestro.py +++ b/maestro.py @@ -106,6 +106,7 @@ TIMESTAMP_SUFFIX = '--timestamped' CONVERT_EXE_PATH = Path('/usr/bin/convert') +MOGRIFY_EXE_PATH = Path('/usr/bin/mogrify') XVFB_EXE_PATH = Path('/usr/bin/xvfb-run') XVFB_SERVER_NUMS = set(range(11, 91)) @@ -125,14 +126,16 @@ TQDM_STYLE = {'bar_format': '{desc} {percentage:3.0f}% {postfix}'} -CONVERT_EXE_PATH_S, XVFB_EXE_PATH_S, FIJI_EXE_PATH_S, FIJI_SCRIPT_BASE_PATH_S, FFMPEG_EXE_PATH_S = \ +CONVERT_EXE_PATH_S, MOGRIFY_EXE_PATH_S, XVFB_EXE_PATH_S, \ +FIJI_EXE_PATH_S, FIJI_SCRIPT_BASE_PATH_S, FFMPEG_EXE_PATH_S = \ map(lambda p: str(p.absolute()), [ -CONVERT_EXE_PATH , XVFB_EXE_PATH , FIJI_EXE_PATH , FIJI_SCRIPT_BASE_PATH , FFMPEG_EXE_PATH +CONVERT_EXE_PATH , MOGRIFY_EXE_PATH, XVFB_EXE_PATH , \ +FIJI_EXE_PATH , FIJI_SCRIPT_BASE_PATH , FFMPEG_EXE_PATH ]) assert CONVERT_EXE_PATH.exists() +assert MOGRIFY_EXE_PATH.exists() assert FFMPEG_EXE_PATH.exists() -assert CONVERT_EXE_PATH.exists() assert XVFB_EXE_PATH.exists() assert FIJI_EXE_PATH.exists() @@ -886,6 +889,7 @@ def encode_movies( def _place_image_files_in_their_respective_well_folders( operetta_images_folder: Path, + abolish_internal_compression: bool = True, tolerate_extra_files: bool = False, retain_original_files: bool = False, ) -> List[Path]: @@ -963,6 +967,13 @@ def assemble_well_folder_name_(row: str, col: str) -> str: assert image_file.exists() transfer_file(image_file, dest_dir_path) + if abolish_internal_compression: + image_file_path = dest_dir_path / image_file + assert image_file_path.exists() + compress_args = ['-compress', 'none'] + image_file_path_s = str(image_file_path.absolute()) + cmd = [MOGRIFY_EXE_PATH_S, *compress_args, image_file_path_s] + sp.run(cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL, check=True) wells_folders_paths = list(sorted([ operetta_images_folder_path / assemble_well_folder_name_(row, col) @@ -1358,7 +1369,7 @@ def remaster( # image pre-processing if before: if 'fix_single_pixel_images' in before: - _check(operetta_export_folder_path, fix_single_pixel_images=True) + _check(operetta_export_folder_path, fix_single_pixel_and_illegible_images=True) if select_best_focus_planes.__name__ in before: select_best_focus_planes(operetta_export_folder_path) @@ -1635,7 +1646,7 @@ def remaster( def _check( operetta_export_folder: Path, - fix_single_pixel_images: bool = False, + fix_single_pixel_and_illegible_images: bool = False, ) -> list: operetta_export_folder_path = Path(operetta_export_folder) @@ -1656,7 +1667,7 @@ def _check( if len(images_not_in_layout) > 0: n_absent = len(images_not_in_layout) - print(f"Warning: There are {n_absent} images not described in the layout metadata!") + print(f"Warning: There are {n_absent} image(s) not described in the layout metadata!") shape_counts = dict(Counter(image_shapes.values())) assert len(shape_counts), "Could not determine the (most prevalent) image shape." @@ -1664,11 +1675,11 @@ def _check( if len(shape_counts) > 1: shape_counts_s = ', '.join([ f"{shape[0]}x{shape[1]}: {count} images" - for shape, count in image_shapes.items() + for shape, count in shape_counts.items() ]) print(f"Warning: There are multiple image shapes: {shape_counts_s}") - if fix_single_pixel_images: + if fix_single_pixel_and_illegible_images: # search for the largest-area shape correct_shape = (0, 0) @@ -1685,7 +1696,7 @@ def _check( cv2.imwrite(str(empty_image_path.absolute()), empty_image.astype(np.uint16)) for image_path, image_shape in tqdm(image_shapes.items(), desc='Info: Fixing ...', **TQDM_STYLE): - if image_shape[0]*image_shape[1] == 1: + if image_shape[0]*image_shape[1] in (0, 1): image_path.rename(str(image_path.absolute()) + '.orig') image_path.symlink_to(empty_image_path.name) @@ -1696,7 +1707,7 @@ def _check( @click.option('--fix-single-pixel-images', is_flag=True, default=False, show_default=True) def check( operetta_export_folder: Path, - fix_single_pixel_images: bool = False, + fix_single_pixel_and_illegible_images: bool = False, ) -> list: ''' Search for: @@ -1704,7 +1715,7 @@ def check( (2) images devoid of respective entries in metadata. ''' - return _check(operetta_export_folder, fix_single_pixel_images) + return _check(operetta_export_folder, fix_single_pixel_and_illegible_images) commands.add_command(check) @@ -1734,11 +1745,13 @@ def folderize( @click.command() @click.argument('operetta_images_folder', type=CLICK_EXISTING_FOLDER_PATH_TYPE) +@click.option('--abolish-internal-compression', is_flag=True, default=True, show_default=True) @click.option('--tolerate-extra-files', is_flag=True, default=False, show_default=True) @click.option('--retain-original-files', is_flag=True, default=False, show_default=True) @click.option('--retain-well-folders', is_flag=True, default=False, show_default=True) def archivize( operetta_images_folder: Path, + abolish_internal_compression: bool = True, tolerate_extra_files: bool = False, retain_original_files: bool = False, retain_well_folders: bool = False, @@ -1752,8 +1765,11 @@ def archivize( compresses the folders. ''' + operetta_images_folder_path = Path(operetta_images_folder) + wells_folders_paths = _place_image_files_in_their_respective_well_folders( - operetta_images_folder=operetta_images_folder, + operetta_images_folder=operetta_images_folder_path, + abolish_internal_compression=abolish_internal_compression, tolerate_extra_files=tolerate_extra_files, retain_original_files=retain_original_files, )