Merge branch 'refactor-api'

README.md

@@ -173,6 +173,8 @@ Tested with python 3.6 and 2.7
 
 - pytorch (>= 1.0.0)
 - torchvision
+- tqdm (>= 4.65.0)
+- h5py (>= 3.7.0)
 - pillow (>= 6.2.0)
 - numpy (>= 1.11)
 - pandas (>= 0.20.3) 
@@ -181,7 +183,7 @@ Tested with python 3.6 and 2.7
 
 Easy installation of dependencies with conda
 ```
-conda install numpy pandas scikit-learn
+conda install numpy pandas scikit-learn h5py tqdm
 conda install -c pytorch pytorch torchvision
 ```
 For more info on installing pytorch for your CUDA version see https://pytorch.org/get-started/locally/

requirements.txt

@@ -1,7 +1,9 @@
 torch >= 1.0.0
 torchvision
+tqdm >= 4.65.0
+h5py >= 3.7.0
 numpy >= 1.11
-pandas
+pandas >= 0.20.3
 scikit-learn >= 0.19.0
 scipy >= 0.17.0
 pillow >= 6.2.0

test/topaz/test_denoise.py

@@ -2,13 +2,9 @@
 
 import numpy as np
 
-from topaz.denoise import (DenoiseNet, DenoiseNet2, GaussianNoise, Identity,
-                           L0Loss, NoiseImages, PairedImages, UDenoiseNet,
-                           UDenoiseNet2, UDenoiseNet3, UDenoiseNet3D,
-                           UDenoiseNetSmall, correct_spatial_covariance,
-                           denoise, denoise_patches, denoise_stack,
-                           estimate_unblur_filter,
-                           estimate_unblur_filter_gaussian, eval_mask_denoise,
-                           eval_noise2noise, gaussian, load_model, lowpass,
-                           spatial_covariance, spatial_covariance_old,
-                           train_mask_denoise, train_noise2noise)
+from topaz.denoise import (Denoise, Denoise3D, GaussianNoise,
+                           correct_spatial_covariance, denoise_image,
+                           denoise_stack, denoise_stream, denoise_tomogram,
+                           denoise_tomogram_stream, estimate_unblur_filter,
+                           estimate_unblur_filter_gaussian, load_model,
+                           lowpass, spatial_covariance)

topaz/algorithms.py

@@ -4,7 +4,7 @@
 from scipy.optimize import linear_sum_assignment
 
 
-def match_coordinates(targets, preds, radius):
+def  match_coordinates(targets:np.ndarray, preds:np.ndarray, radius:float) -> tuple[np.ndarray, np.ndarray]:
     d2 = np.sum((preds[:,np.newaxis] - targets[np.newaxis])**2, 2)
     cost = d2 - radius*radius
     cost[cost > 0] = 0
@@ -22,8 +22,8 @@ def match_coordinates(targets, preds, radius):
     return assignment, dist
 
 
-def non_maximum_suppression(x, r, threshold=-np.inf):
-    ## enumerate coordinate deltas within d
+def non_maximum_suppression(x, r:int, threshold:float=-np.inf) -> tuple[np.ndarray, np.ndarray]:
+    ## enumerate coordinate deltas within radius/distance r
     width = r
     ii,jj = np.meshgrid(np.arange(-width,width+1), np.arange(-width,width+1))
     mask = (ii**2 + jj**2) <= r*r
@@ -63,9 +63,9 @@ def non_maximum_suppression(x, r, threshold=-np.inf):
     return scores[:j], coords[:j]
 
 
-def non_maximum_suppression_3d(x, d, scale=1.0, threshold=-np.inf):
-    ## enumerate coordinate deltas within d
-    r = scale*d/2
+def non_maximum_suppression_3d(x:np.ndarray, r:int, scale:float=1.0, threshold:float=-np.inf) -> tuple[np.ndarray, np.ndarray]:
+    ## enumerate coordinate deltas within (possibly scaled) radius/distance r
+    r = scale*r
     width = int(np.ceil(r))
     A = np.arange(-width,width+1)
     ii,jj,kk = np.meshgrid(A, A, A)
@@ -100,11 +100,4 @@ def non_maximum_suppression_3d(x, d, scale=1.0, threshold=-np.inf):
             for delta in coord_deltas:
                 S.add(i + delta)
 
-    return scores[:j], coords[:j]
-
-
-
-
-
-
-
+    return scores[:j], coords[:j]

topaz/commands/boxes_to_coordinates.py

@@ -1,10 +1,8 @@
-from __future__ import print_function,division
+from __future__ import division, print_function
 
-import sys
-import os
-import pandas as pd
-from PIL import Image
-import glob
+import argparse
+
+from topaz.utils.conversions import file_boxes_to_coordinates
 
 name = 'boxes_to_coordinates'
 help = 'convert .box format coordinates to tab delimited coordinates table'
@@ -15,41 +13,15 @@ def add_arguments(parser):
     parser.add_argument('--imagedir', help='directory of images. only required to invert the y-axis - necessary for particles picked on .tiff images in EMAN2')
     parser.add_argument('--image-ext', default='tiff', help='image format extension, * corresponds to matching the first image file with the same name as the box file (default: tiff)')
     parser.add_argument('-o', '--output', help='destination file (default: stdout)')
-
-def main(args):
-    from topaz.utils.conversions import boxes_to_coordinates
-    from topaz.utils.data.loader import load_image
-
-    tables = []
-
-    invert_y = args.invert_y
-
-    for path in args.files:
-        if os.path.getsize(path) == 0:
-            continue
-
-        shape = None
-        image_name = os.path.splitext(os.path.basename(path))[0]
-        if invert_y:
-            impath = os.path.join(args.imagedir, image_name) + '.' + args.image_ext
-            # use glob incase image_ext is '*'
-            impath = glob.glob(impath)[0]
-            im = load_image(impath)
-            shape = (im.height,im.width)
-
-        box = pd.read_csv(path, sep='\t', header=None).values
-
-        coords = boxes_to_coordinates(box, shape=shape, invert_y=invert_y, image_name=image_name)
-
-        tables.append(coords)
-
-    table = pd.concat(tables, axis=0)
-
-    output = sys.stdout
-    if args.output is not None:
-        output = args.output
-    table.to_csv(output, sep='\t', index=False)
-
-
+    return parser
 
 
+def main(args):
+    file_boxes_to_coordinates(args.files, args.imagedir, args.image_ext, args.invert_y, args.output)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(help)
+    parser = add_arguments(parser)
+    args = parser.parse_args()
+    main(args)

topaz/commands/convert.py

@@ -113,7 +113,7 @@ def main(args):
         else:
             try:
                 to_form = file_utils.detect_format(output_path)
-            except file_utils.UnkownFormatError as e:
+            except file_utils.UnknownFormatError as e:
                 print('Error: unrecognized output coordinates file extension ('+e.ext+')', file=sys.stderr)
                 sys.exit(1)
     if verbose > 0:

topaz/commands/coordinates_to_boxes.py

@@ -1,15 +1,8 @@
-from __future__ import print_function,division
+from __future__ import division, print_function
 
-import sys
-import os
-import numpy as np
-import pandas as pd
-from PIL import Image
-import glob
-
-from topaz.utils.conversions import coordinates_to_boxes
-from topaz.utils.data.loader import load_image
+import argparse
 
+from topaz.utils.conversions import file_coordinates_to_boxes
 
 name = 'coordinates_to_boxes'
 help = 'convert coordinates table to .box format files per image'
@@ -23,48 +16,15 @@ def add_arguments(parser):
     parser.add_argument('--invert-y', action='store_true', help='invert (mirror) the y-axis particle coordinates. appears to be necessary for .tiff compatibility with EMAN2')
     parser.add_argument('--imagedir', help='directory of images. only required to invert the y-axis - necessary for particles picked on .tiff images in EMAN2')
     parser.add_argument('--image-ext', default='tiff', help='image format extension, * corresponds to matching the first image file with the same name as the box file (default: tiff)')
-
     return parser
 
 
 def main(args):
-    dfs = []
-    for path in args.paths:
-        coords = pd.read_csv(path, sep='\t')
-        dfs.append(coords)
-    coords = pd.concat(dfs, axis=0)
-
-    coords = coords.drop_duplicates()
-
-    if not os.path.exists(args.destdir):
-        os.makedirs(args.destdir)
-
-    invert_y = args.invert_y
-
-    for image_name,group in coords.groupby('image_name'):
-        path = args.destdir + '/' + image_name + '.box'
-
-        shape = None
-        if invert_y:
-            impath = os.path.join(args.imagedir, image_name) + '.' + args.image_ext
-            # use glob incase image_ext is '*'
-            impath = glob.glob(impath)[0]
-            im = load_image(impath)
-            shape = (im.height,im.width)
-
-        xy = group[['x_coord', 'y_coord']].values.astype(np.int32)
-
-        boxes = coordinates_to_boxes(xy, args.boxsize, args.boxsize, shape=shape, invert_y=invert_y)
-        boxes = pd.DataFrame(boxes)
-
-        boxes.to_csv(path, sep='\t', header=False, index=False)
+    file_coordinates_to_boxes(args.files, args.destdir, args.boxsize, args.invert_y, args.imagedir, args.image_ext)
 
 
 if __name__ == '__main__':
-    import argparse
     parser = argparse.ArgumentParser('Script for converting coordinates for images in one file to multiple files')
-    add_arguments(parser)
+    parser = add_arguments(parser)
     args = parser.parse_args()
     main(args)
-
-

topaz/commands/coordinates_to_eman2_json.py

@@ -1,16 +1,8 @@
-from __future__ import print_function,division
+from __future__ import division, print_function
 
-import sys
-import os
-import numpy as np
-import pandas as pd
-import json
-from PIL import Image
-import glob
-
-from topaz.utils.conversions import coordinates_to_eman2_json
-from topaz.utils.data.loader import load_image
+import argparse
 
+from topaz.utils.conversions import file_coordinates_to_eman2_json
 
 name = 'coordinates_to_eman2_json'
 help = 'convert coordinates table to EMAN2 json format files per image'
@@ -28,41 +20,11 @@ def add_arguments(parser):
 
 
 def main(args):
-    dfs = []
-    for path in args.paths:
-        coords = pd.read_csv(path, sep='\t')
-        dfs.append(coords)
-    coords = pd.concat(dfs, axis=0)
-
-    coords = coords.drop_duplicates()
-    print(len(coords))
-
-    if not os.path.exists(args.destdir):
-        os.makedirs(args.destdir)
-
-    invert_y = args.invert_y
-
-    for image_name,group in coords.groupby('image_name'):
-        path = args.destdir + '/' + image_name + '_info.json'
-
-        shape = None
-        if invert_y:
-            impath = os.path.join(args.imagedir, image_name) + '.' + args.image_ext
-            # use glob incase image_ext is '*'
-            impath = glob.glob(impath)[0]
-            im = load_image(impath)
-            shape = (im.height,im.width)
-
-        xy = group[['x_coord','y_coord']].values.astype(int)
-        boxes = coordinates_to_eman2_json(xy, shape=shape, invert_y=invert_y)
-
-        with open(path, 'w') as f:
-            json.dump({'boxes': boxes}, f, indent=0)
+    file_coordinates_to_eman2_json(args.paths, args.destdir, args.invert_y, args.imagedir, args.image_ext)
 
 
 if __name__ == '__main__':
-    import argparse
     parser = argparse.ArgumentParser(help)
-    add_arguments(parser)
+    parser = add_arguments(parser)
     args = parser.parse_args()
     main(args)