From 2a7e2a7a10aadc8286507f266c7aa0ddbcbccc20 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 10:51:38 -0500 Subject: [PATCH 01/38] generalize pixeltype --- .../bigstitcher/spark/SparkAffineFusion.java | 37 +++++++++---------- .../spark/SparkNonRigidFusion.java | 32 +++++----------- .../bigstitcher/spark/util/Import.java | 10 ++--- 3 files changed, 31 insertions(+), 48 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 0931585..72fbf76 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -21,7 +21,6 @@ */ package net.preibisch.bigstitcher.spark; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.net.URI; @@ -37,7 +36,6 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.janelia.scicomp.n5.zstandard.ZstandardCompression; @@ -54,7 +52,6 @@ import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.bigstitcher.spark.util.N5Util; import net.preibisch.bigstitcher.spark.util.Spark; -import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; import net.preibisch.mvrecon.process.export.ExportN5Api; @@ -69,6 +66,11 @@ public class SparkAffineFusion extends AbstractSelectableViews implements Callable, Serializable { + public static enum DataTypeFusion + { + UINT8, UINT16, FLOAT32 + } + private static final long serialVersionUID = -6103761116219617153L; @Option(names = { "-o", "--n5Path" }, required = true, description = "N5/ZARR/HDF5 basse path for saving (must be combined with the option '-d' or '--bdv'), e.g. -o /home/fused.n5 or e.g. s3://myBucket/data.n5") @@ -109,12 +111,9 @@ public class SparkAffineFusion extends AbstractSelectableViews implements Callab @Option(names = { "--anisotropyFactor" }, description = "define the anisotropy factor if preserveAnisotropy is set to true (default: compute from data)") private double anisotropyFactor = Double.NaN; - // TODO: make a variable just as -s is - @Option(names = { "--UINT16" }, description = "save as UINT16 [0...65535], if you choose it you must define min and max intensity (default: fuse as 32 bit float)") - private boolean uint16 = false; - - @Option(names = { "--UINT8" }, description = "save as UINT8 [0...255], if you choose it you must define min and max intensity (default: fuse as 32 bit float)") - private boolean uint8 = false; + @Option(names = {"-p", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") + private DataTypeFusion dataTypeFusion = null; @Option(names = { "--minIntensity" }, description = "min intensity for scaling values to the desired range (required for UINT8 and UINT16), e.g. 0.0") private Double minIntensity = null; @@ -162,7 +161,7 @@ public Void call() throws Exception return null; } - Import.validateInputParameters(uint8, uint16, minIntensity, maxIntensity); + Import.validateInputParameters( dataTypeFusion, minIntensity, maxIntensity); final SpimData2 dataGlobal = this.loadSpimData2(); @@ -192,12 +191,12 @@ public Void call() throws Exception final DataType dataType; - if ( uint8 ) + if ( dataTypeFusion == DataTypeFusion.UINT8 ) { System.out.println( "Fusing to UINT8, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); dataType = DataType.UINT8; } - else if ( uint16 ) + else if ( dataTypeFusion == DataTypeFusion.UINT16 ) { System.out.println( "Fusing to UINT16, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); dataType = DataType.UINT16; @@ -262,11 +261,11 @@ else if ( this.downsampling != null ) // TODO: expose final Compression compression = new ZstandardCompression( 3 );// new GzipCompression( 1 ); - final double minIntensity = ( uint8 || uint16 ) ? this.minIntensity : 0; + final double minIntensity = ( dataTypeFusion != DataTypeFusion.FLOAT32 ) ? this.minIntensity : 0; final double range; - if ( uint8 ) + if ( dataTypeFusion == DataTypeFusion.UINT8 ) range = ( this.maxIntensity - this.minIntensity ) / 255.0; - else if ( uint16 ) + else if ( dataTypeFusion == DataTypeFusion.UINT16 ) range = ( this.maxIntensity - this.minIntensity ) / 65535.0; else range = 0; @@ -363,8 +362,8 @@ else if ( uint16 ) n5Dataset, storageType, serializedViewIds, - uint8, - uint16, + dataTypeFusion == DataTypeFusion.UINT8, + dataTypeFusion == DataTypeFusion.UINT16, maskOff, blockSize ) ); else @@ -378,8 +377,8 @@ else if ( uint16 ) bdvString, storageType, serializedViewIds, - uint8, - uint16, + dataTypeFusion == DataTypeFusion.UINT8, + dataTypeFusion == DataTypeFusion.UINT16, minIntensity, range, blockSize, diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkNonRigidFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkNonRigidFusion.java index 01aebbe..ff4c08e 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkNonRigidFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkNonRigidFusion.java @@ -58,6 +58,7 @@ import net.imglib2.type.numeric.real.FloatType; import net.imglib2.util.Intervals; import net.imglib2.util.Util; +import net.preibisch.bigstitcher.spark.SparkAffineFusion.DataTypeFusion; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; import net.preibisch.bigstitcher.spark.util.BDVSparkInstantiateViewSetup; import net.preibisch.bigstitcher.spark.util.Import; @@ -112,11 +113,9 @@ public class SparkNonRigidFusion extends AbstractSelectableViews implements Call @Option(names = { "-ip", "--interestPoints" }, required = true, description = "provide a list of corresponding interest points to be used for the fusion (e.g. -ip 'beads' -ip 'nuclei'") private ArrayList interestPoints = null; - @Option(names = { "--UINT16" }, description = "save as UINT16 [0...65535], if you choose it you must define min and max intensity (default: fuse as 32 bit float)") - private boolean uint16 = false; - - @Option(names = { "--UINT8" }, description = "save as UINT8 [0...255], if you choose it you must define min and max intensity (default: fuse as 32 bit float)") - private boolean uint8 = false; + @Option(names = {"-p", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") + private DataTypeFusion dataTypeFusion = null; @Option(names = { "--minIntensity" }, description = "min intensity for scaling values to the desired range (required for UINT8 and UINT16), e.g. 0.0") private Double minIntensity = null; @@ -147,13 +146,7 @@ public Void call() throws Exception return null; } - Import.validateInputParameters(uint8, uint16, minIntensity, maxIntensity); - - if ( StorageFormat.HDF5.equals( storageType ) && bdvString != null && !uint16 ) - { - System.out.println( "BDV-compatible HDF5 only supports 16-bit output for now. Please use '--UINT16' flag for fusion." ); - System.exit( 0 ); - } + Import.validateInputParameters(dataTypeFusion, minIntensity, maxIntensity); final SpimData2 dataGlobal = this.loadSpimData2(); @@ -191,17 +184,12 @@ public Void call() throws Exception final DataType dataType; - if ( uint8 ) + if ( dataTypeFusion == DataTypeFusion.UINT8 ) { System.out.println( "Fusing to UINT8, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); dataType = DataType.UINT8; } - else if ( uint16 && bdvString != null && StorageFormat.HDF5.equals( storageType ) ) - { - System.out.println( "Fusing to INT16 (for BDV compliance, which is treated as UINT16), min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); - dataType = DataType.INT16; - } - else if ( uint16) + else if ( dataTypeFusion == DataTypeFusion.UINT16 ) { System.out.println( "Fusing to UINT16, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); dataType = DataType.UINT16; @@ -227,8 +215,8 @@ else if ( uint16) final Compression compression = new ZstandardCompression( 3 ); final ArrayList< String > labels = new ArrayList<>(interestPoints); - final boolean uint8 = this.uint8; - final boolean uint16 = this.uint16; + final boolean uint8 = (dataTypeFusion == DataTypeFusion.UINT8); + final boolean uint16 = (dataTypeFusion == DataTypeFusion.UINT16); final double minIntensity = (uint8 || uint16 ) ? this.minIntensity : 0; final double range; if ( uint8 ) @@ -296,7 +284,7 @@ else if ( uint16 ) downsamplings, viewId, n5PathURI, - xmlOutURI, + xmloutURI, instantiate ) == null ) { System.out.println( "Failed to write metadata for '" + n5Dataset + "'." ); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java index bd36e23..fb8da0b 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java @@ -29,6 +29,7 @@ import mpicbg.spim.data.SpimData; import mpicbg.spim.data.sequence.ViewDescription; import mpicbg.spim.data.sequence.ViewId; +import net.preibisch.bigstitcher.spark.SparkAffineFusion.DataTypeFusion; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; import net.preibisch.mvrecon.process.boundingbox.BoundingBoxTools; @@ -65,17 +66,12 @@ public static BoundingBox getBoundingBox( } public static void validateInputParameters( - final boolean uint8, - final boolean uint16, + final DataTypeFusion datatype, final Double minIntensity, final Double maxIntensity ) throws IllegalArgumentException { - if ( uint8 && uint16 ) { - throw new IllegalArgumentException( "Please only select UINT8, UINT16 or nothing (FLOAT32)." ); - } - - if ( ( uint8 || uint16 ) && (minIntensity == null || maxIntensity == null ) ) { + if ( ( datatype == DataTypeFusion.UINT8 || datatype == DataTypeFusion.UINT16 ) && (minIntensity == null || maxIntensity == null ) ) { throw new IllegalArgumentException( "When selecting UINT8 or UINT16 you need to specify minIntensity and maxIntensity." ); } } From 7d86ea2106900da3ad0ae08cd4865af7b21e22e4 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 11:33:28 -0500 Subject: [PATCH 02/38] added first code for creating fusion container --- .../spark/CreateFusionContainer.java | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java new file mode 100644 index 0000000..64d6195 --- /dev/null +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -0,0 +1,135 @@ +package net.preibisch.bigstitcher.spark; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; + +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; + +import mpicbg.spim.data.SpimDataException; +import mpicbg.spim.data.sequence.ViewId; +import net.preibisch.bigstitcher.spark.SparkAffineFusion.DataTypeFusion; +import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractBasic; +import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; +import net.preibisch.bigstitcher.spark.util.Import; +import net.preibisch.mvrecon.fiji.spimdata.SpimData2; +import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class CreateFusionContainer extends AbstractBasic implements Callable, Serializable +{ + private static final long serialVersionUID = -9140450542904228386L; + + @Option(names = { "-o", "--outputPath" }, required = true, description = "OME-ZARR path for saving, e.g. -o /home/fused.zarr, file:/home/fused.n5 or e.g. s3://myBucket/data.zarr") + private String outputPathURIString = null; + + @Option(names = {"-s", "--storage"}, defaultValue = "ZARR", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + description = "Dataset storage type, currently supported OME-ZARR, N5, and ONLY for local, multithreaded Spark HDF5 (default: OME-ZARR)") + private StorageFormat storageType = null; + + @Option(names = {"-ch", "--numChannels" }, description = "number of fused channels in the output container (default: as many as in the XML)") + private Integer numChannels = null; + + @Option(names = {"-tp", "--numTimepoints" }, description = "number of fused timepoints in the output container (default: as many as in the XML)") + private Integer numTimepoints = null; + + @Option(names = "--blockSize", description = "blockSize (default: 128,128,128)") + private String blockSizeString = "128,128,128"; + + @Option(names = {"-p", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") + private DataTypeFusion dataTypeFusion = null; + + @Option(names = { "--bdv" }, required = false, description = "Write a BigDataViewer-compatible dataset (default: false)") + private boolean bdv = false; + + @Option(names = { "-xo", "--xmlout" }, required = false, description = "path to the new BigDataViewer xml project (only valid if --bdv was selected), " + + "e.g. -xo /home/project.xml or -xo s3://myBucket/project.xml (default: dataset.xml in basepath for H5, dataset.xml one directory level above basepath for N5)") + private String xmlOutURIString = null; + + @Option(names = { "-b", "--boundingBox" }, description = "fuse a specific bounding box listed in the XML (default: fuse everything)") + private String boundingBoxName = null; + + @Option(names = { "--multiRes" }, description = "Automatically create a multi-resolution pyramid (default: false)") + private boolean multiRes = false; + + @Option(names = { "-ds", "--downsampling" }, split = ";", required = false, description = "Manually define steps to create of a multi-resolution pyramid (e.g. -ds 2,2,1; 2,2,1; 2,2,2; 2,2,2)") + private List downsampling = null; + + @Option(names = { "--preserveAnisotropy" }, description = "preserve the anisotropy of the data (default: false)") + private boolean preserveAnisotropy = false; + + @Option(names = { "--anisotropyFactor" }, description = "define the anisotropy factor if preserveAnisotropy is set to true (default: compute from data)") + private double anisotropyFactor = Double.NaN; + + + @Override + public Void call() throws Exception + { + if (dryRun) + { + System.out.println( "dry-run not supported for CreateFusionContainer."); + System.exit( 0 ); + } + + if ( this.bdv && xmlOutURIString == null ) + { + System.out.println( "Please specify the output XML for the BDV dataset: -xo"); + return null; + } + + final SpimData2 dataGlobal = this.loadSpimData2(); + + if ( dataGlobal == null ) + return null; + + final ArrayList< ViewId > viewIdsGlobal = Import.getViewIds( dataGlobal ); + + if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) + return null; + + final int numTimepointsXML = dataGlobal.getSequenceDescription().getTimePoints().getTimePointsOrdered().size(); + final int numChannelsXML = dataGlobal.getSequenceDescription().getAllChannelsOrdered().size(); + + System.out.println( "XML project contains " + numChannelsXML + " channels, " + numTimepointsXML + " timepoints." ); + + if ( numChannels == null ) + numChannels = numChannelsXML; + + if ( numTimepoints == null ) + numTimepoints = numTimepointsXML; + + if ( numChannels < numChannelsXML ) + System.out.println( "WARNING: you selected to fuse LESS channels than present in the data. This works, but you will need specify the content manually."); + else if ( numChannels > numChannelsXML ) + System.out.println( "WARNING: you selected to fuse MORE channels than present in the data. This works, but you will need specify the content manually."); + + if ( numTimepoints < numTimepointsXML ) + System.out.println( "WARNING: you selected to fuse LESS timepoints than present in the data. This works, but you will need specify the content manually."); + else if ( numTimepoints > numTimepointsXML ) + System.out.println( "WARNING: you selected to fuse MORE timepoints than present in the data. This works, but you will need specify the content manually."); + + + + BoundingBox boundingBox = Import.getBoundingBox( dataGlobal, viewIdsGlobal, boundingBoxName ); + + // TODO Auto-generated method stub + return null; + } + + public static void main(final String... args) throws SpimDataException + { + + //final XmlIoSpimData io = new XmlIoSpimData(); + //final SpimData spimData = io.load( "/Users/preibischs/Documents/Microscopy/Stitching/Truman/standard/output/dataset.xml" ); + //BdvFunctions.show( spimData ); + //SimpleMultiThreading.threadHaltUnClean(); + + System.out.println(Arrays.toString(args)); + + System.exit(new CommandLine(new CreateFusionContainer()).execute(args)); + } +} From 87b7fe94f830c7ef880a926f9fe1b4ac2b768053 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 13:23:24 -0500 Subject: [PATCH 03/38] working on downsampling and other parameters --- .../spark/CreateFusionContainer.java | 72 ++++++++++++++++++- .../bigstitcher/spark/util/Downsampling.java | 12 +--- .../bigstitcher/spark/util/Import.java | 8 ++- 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 64d6195..2b375a6 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -1,6 +1,7 @@ package net.preibisch.bigstitcher.spark; import java.io.Serializable; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -10,14 +11,20 @@ import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.sequence.ViewId; +import net.imglib2.FinalDimensions; +import net.imglib2.FinalInterval; +import net.imglib2.util.Util; import net.preibisch.bigstitcher.spark.SparkAffineFusion.DataTypeFusion; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractBasic; -import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; +import net.preibisch.bigstitcher.spark.util.Downsampling; import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; +import net.preibisch.mvrecon.process.export.ExportN5Api; +import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; import picocli.CommandLine; import picocli.CommandLine.Option; +import util.URITools; public class CreateFusionContainer extends AbstractBasic implements Callable, Serializable { @@ -65,6 +72,7 @@ public class CreateFusionContainer extends AbstractBasic implements Callable numChannelsXML ) else if ( numTimepoints > numTimepointsXML ) System.out.println( "WARNING: you selected to fuse MORE timepoints than present in the data. This works, but you will need specify the content manually."); + this.outPathURI = URITools.toURI( outputPathURIString ); + System.out.println( "ZARR/N5/HDF5 container: " + outPathURI ); + if ( this.bdv ) + { + this.xmlOutURI = URITools.toURI( xmlOutURIString ); + System.out.println( "XML: " + xmlOutURI ); + + if ( storageType == StorageFormat.ZARR ) + { + System.out.println( "BDV project for OME-ZARR not yet supported (but very soon!)" ); + return null; + } + } BoundingBox boundingBox = Import.getBoundingBox( dataGlobal, viewIdsGlobal, boundingBoxName ); + final long[] minBB = boundingBox.minAsLongArray(); + final long[] maxBB = boundingBox.maxAsLongArray(); + + if ( preserveAnisotropy ) + { + System.out.println( "Preserving anisotropy."); + + if ( Double.isNaN( anisotropyFactor ) ) + { + anisotropyFactor = TransformationTools.getAverageAnisotropyFactor( dataGlobal, viewIdsGlobal ); + + System.out.println( "Anisotropy factor [computed from data]: " + anisotropyFactor ); + } + else + { + System.out.println( "Anisotropy factor [provided]: " + anisotropyFactor ); + } + + // prepare downsampled boundingbox + minBB[ 2 ] = Math.round( Math.floor( minBB[ 2 ] / anisotropyFactor ) ); + maxBB[ 2 ] = Math.round( Math.ceil( maxBB[ 2 ] / anisotropyFactor ) ); + + boundingBox = new BoundingBox( new FinalInterval(minBB, maxBB) ); + + System.out.println( "Adjusted bounding box (anisotropy preserved: " + Util.printInterval( boundingBox ) ); + } + + final int[] blockSize = Import.csvStringToIntArray( blockSizeString ); + + System.out.println( "Fusion target: " + boundingBox.getTitle() + ": " + Util.printInterval( boundingBox ) + " with blocksize " + Util.printCoordinates( blockSize ) ); + + // + // set up downsampling (if wanted) + // + if ( !Downsampling.testDownsamplingParameters( this.multiRes, this.downsampling ) ) + return null; + + final int[][] downsamplings; + + if ( multiRes ) + downsamplings = ExportN5Api.estimateMultiResPyramid( new FinalDimensions( boundingBox ), anisotropyFactor ); + else if ( this.downsampling != null ) + downsamplings = Import.csvStringListToDownsampling( this.downsampling ); + else + downsamplings = new int[][]{{ 1, 1, 1 }}; + + System.out.println( "The following downsampling pyramid will be created:" ); + System.out.println( Arrays.deepToString( downsamplings ) ); + // TODO Auto-generated method stub return null; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Downsampling.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Downsampling.java index 2b9dfaa..50f7b5d 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Downsampling.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Downsampling.java @@ -228,7 +228,7 @@ else if ( datatype == DataType.INT16 ) return true; } - public static boolean testDownsamplingParameters( final boolean multiRes, final List downsampling, final String dataset ) + public static boolean testDownsamplingParameters( final boolean multiRes, final List downsampling ) { // no not create multi-res pyramid if ( !multiRes && downsampling == null ) @@ -240,16 +240,6 @@ public static boolean testDownsamplingParameters( final boolean multiRes, final return false; } - // non-bdv multi-res dataset - if ( dataset != null ) - { - if ( !dataset.endsWith("/s0") ) - { - System.out.println( "In order to create a multi-resolution pyramid for a non-BDV dataset, the dataset must end with '/s0', right not it is '" + dataset + "'."); - return false; - } - } - return true; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java index fb8da0b..915b9cd 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java @@ -254,14 +254,16 @@ public static int[][] csvStringToDownsampling(final String csvString) { } /** - * converts a List of Strings like '[1,1,1][ 2,2,1][ 4,4,1 ][ 8,8,2] to downsampling levels in int[][] + * converts a List of Strings (relative steps) like '[2,2,1][2,2,2] to downsampling levels in int[][], e.g. [1,1,1][2,2,1][4,4,2] * @param csvString * @return */ public static int[][] csvStringListToDownsampling(final List csvString) { - final int[][] downsampling = new int[csvString.size()][]; - for ( int i = 0; i < csvString.size(); ++i ) + final int[][] downsampling = new int[csvString.size() + 1][]; + downsampling[ i ] = new int[] { 1, 1, 1 }; + + for ( int i = 1; i <= csvString.size(); ++i ) downsampling[ i ] = Arrays.stream(csvString.get( i ).split(",")).map( st -> st.trim() ).mapToInt(Integer::parseInt).toArray(); return downsampling; From 155db7cf145d24b3d8f5742fc3d3b688e258f16e Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 15:13:04 -0500 Subject: [PATCH 04/38] fix downsampling string parsing --- .../bigstitcher/spark/util/Import.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java index 915b9cd..a247a36 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Import.java @@ -254,17 +254,34 @@ public static int[][] csvStringToDownsampling(final String csvString) { } /** - * converts a List of Strings (relative steps) like '[2,2,1][2,2,2] to downsampling levels in int[][], e.g. [1,1,1][2,2,1][4,4,2] + * converts a List of Strings like '[1,1,1][ 2,2,1][ 4,4,1 ][ 8,8,2] to downsampling levels in int[][] * @param csvString * @return */ - public static int[][] csvStringListToDownsampling(final List csvString) { - - final int[][] downsampling = new int[csvString.size() + 1][]; - downsampling[ i ] = new int[] { 1, 1, 1 }; - - for ( int i = 1; i <= csvString.size(); ++i ) + public static int[][] csvStringListToDownsampling(final List csvString) + { + if ( csvString == null || csvString.size() < 1 ) + { + System.out.println( "List of strings for downsampling is empty/null."); + return null; + } + + final int[][] downsampling = new int[csvString.size()][]; + for ( int i = 0; i < csvString.size(); ++i ) + { downsampling[ i ] = Arrays.stream(csvString.get( i ).split(",")).map( st -> st.trim() ).mapToInt(Integer::parseInt).toArray(); + if ( downsampling[ i ].length != 3 ) + { + System.out.println( "dimensions of downsampling entry is not 3: " + csvString.get( i ) ); + return null; + } + } + + if ( !Arrays.equals( downsampling[ 0 ], new int[] { 1, 1, 1 })) + { + System.out.println( "first entry is not [1,1,1], but must be: " + csvString.get( 0 ) ); + return null; + } return downsampling; } From 3f0b8aac01bbc93cbc8f513704026094335be99f Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 16:36:38 -0500 Subject: [PATCH 05/38] update mvr --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6dc705e..7aa4ef4 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 10.6.3 2.3.5 - 5.0.1 + 5.0.3 2.3.1 From 0714354530e947f9690c2e93de842087c4cc88b6 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 16:36:45 -0500 Subject: [PATCH 06/38] progress towards creating fused dataset containers --- .../spark/CreateFusionContainer.java | 205 +++++++++++++++++- 1 file changed, 197 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 2b375a6..5ecc97c 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -1,27 +1,48 @@ package net.preibisch.bigstitcher.spark; +import java.io.File; import java.io.Serializable; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; - +import java.util.function.Function; + +import org.janelia.saalfeldlab.n5.Bzip2Compression; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.Lz4Compression; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.XzCompression; +import org.janelia.saalfeldlab.n5.blosc.BloscCompression; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; +import bdv.util.MipmapTransforms; import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.sequence.ViewId; +import mpicbg.spim.data.sequence.VoxelDimensions; import net.imglib2.FinalDimensions; import net.imglib2.FinalInterval; +import net.imglib2.realtransform.AffineTransform3D; import net.imglib2.util.Util; import net.preibisch.bigstitcher.spark.SparkAffineFusion.DataTypeFusion; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractBasic; import net.preibisch.bigstitcher.spark.util.Downsampling; import net.preibisch.bigstitcher.spark.util.Import; +import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; +import net.preibisch.mvrecon.fiji.spimdata.imgloaders.OMEZarrAttibutes; import net.preibisch.mvrecon.process.export.ExportN5Api; import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; +import net.preibisch.mvrecon.process.n5api.N5ApiTools; +import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; import picocli.CommandLine; import picocli.CommandLine.Option; import util.URITools; @@ -30,6 +51,8 @@ public class CreateFusionContainer extends AbstractBasic implements Callable downsampling = null; @Option(names = { "--preserveAnisotropy" }, description = "preserve the anisotropy of the data (default: false)") @@ -89,6 +119,15 @@ public Void call() throws Exception return null; } + this.outPathURI = URITools.toURI( outputPathURIString ); + System.out.println( "ZARR/N5/HDF5 container: " + outPathURI ); + + if ( storageType == StorageFormat.HDF5 && URITools.isFile( outPathURI ) ) + { + System.out.println( "HDF5 only supports local storage, but --outputPath=" + outPathURI ); + return null; + } + final SpimData2 dataGlobal = this.loadSpimData2(); if ( dataGlobal == null ) @@ -120,9 +159,6 @@ else if ( numChannels > numChannelsXML ) else if ( numTimepoints > numTimepointsXML ) System.out.println( "WARNING: you selected to fuse MORE timepoints than present in the data. This works, but you will need specify the content manually."); - this.outPathURI = URITools.toURI( outputPathURIString ); - System.out.println( "ZARR/N5/HDF5 container: " + outPathURI ); - if ( this.bdv ) { this.xmlOutURI = URITools.toURI( xmlOutURIString ); @@ -161,15 +197,55 @@ else if ( numTimepoints > numTimepointsXML ) boundingBox = new BoundingBox( new FinalInterval(minBB, maxBB) ); - System.out.println( "Adjusted bounding box (anisotropy preserved: " + Util.printInterval( boundingBox ) ); + System.out.println( "Adjusted bounding box (anisotropy preserved): " + Util.printInterval( boundingBox ) ); } final int[] blockSize = Import.csvStringToIntArray( blockSizeString ); System.out.println( "Fusion target: " + boundingBox.getTitle() + ": " + Util.printInterval( boundingBox ) + " with blocksize " + Util.printCoordinates( blockSize ) ); + // compression and data type + final Compression compression; + + //Lz4, Gzip, Zstandard, Blosc, Bzip2, Xz, Raw }; + if ( this.compression == Compressions.Lz4 ) + compression = new Lz4Compression(); + else if ( this.compression == Compressions.Gzip ) + compression = new GzipCompression( compressionLevel == null ? 1 : compressionLevel ); + else if ( this.compression == Compressions.Zstandard ) + compression = new ZstandardCompression( compressionLevel == null ? 3 : compressionLevel ); + else if ( this.compression == Compressions.Blosc ) + compression = new BloscCompression(); + else if ( this.compression == Compressions.Bzip2 ) + compression = new Bzip2Compression(); + else if ( this.compression == Compressions.Xz ) + compression = new XzCompression( compressionLevel == null ? 6 : compressionLevel ); + else if ( this.compression == Compressions.Raw ) + compression = new RawCompression(); + else + compression = null; + + System.out.println( "Compression: " + this.compression ); + System.out.println( "Compression level: " + ( compressionLevel == null ? "default" : compressionLevel ) ); + + final DataType dt; + + if ( dataTypeFusion == DataTypeFusion.UINT8 ) + dt = DataType.UINT8; + else if ( dataTypeFusion == DataTypeFusion.UINT16 ) + dt = DataType.UINT16; + else if ( dataTypeFusion == DataTypeFusion.FLOAT32 ) + dt = DataType.FLOAT32; + else + dt = null; + + System.out.println( "Data type: " + dt ); + + if ( dt == null || compression == null ) + return null; + // - // set up downsampling (if wanted) + // set up downsampling // if ( !Downsampling.testDownsamplingParameters( this.multiRes, this.downsampling ) ) return null; @@ -183,10 +259,123 @@ else if ( this.downsampling != null ) else downsamplings = new int[][]{{ 1, 1, 1 }}; + if ( downsamplings == null ) + return null; + System.out.println( "The following downsampling pyramid will be created:" ); System.out.println( Arrays.deepToString( downsamplings ) ); - // TODO Auto-generated method stub + // + // set up container and metadata + // + final N5Writer driverVolumeWriter; + //MultiResolutionLevelInfo[] mrInfo; + + System.out.println(); + System.out.println( "Setting up container and metadata in '" + outPathURI + "' ... " ); + + // init base folder and writer + if ( storageType == StorageFormat.HDF5 ) + { + final File dir = new File( URITools.fromURI( outPathURI ) ).getParentFile(); + if ( !dir.exists() ) + dir.mkdirs(); + driverVolumeWriter = new N5HDF5Writer( URITools.fromURI( outPathURI ) ); + } + else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) + { + driverVolumeWriter = URITools.instantiateN5Writer( storageType, outPathURI ); + } + else + { + System.out.println( "Unsupported format: " + storageType ); + return null; + + } + + // setup datasets and metadata + if ( bdv ) + { + // TODO: set extra attributes to load the state + } + else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export + { + IOFunctions.println( "Creating 5D OME-ZARR metadata for '" + outPathURI + "' ... " ); + + final long[] dim3d = boundingBox.dimensionsAsLongArray(); + + final long[] dim = new long[] { dim3d[ 0 ], dim3d[ 1 ], dim3d[ 2 ], numChannels, numTimepoints }; + final int[] blockSize5d = new int[] { blockSize[ 0 ], blockSize[ 1 ], blockSize[ 2 ], 1, 1 }; + final int[][] ds = new int[ downsamplings.length ][]; + for ( int d = 0; d < ds.length; ++d ) + ds[ d ] = new int[] { downsamplings[ d ][ 0 ], downsamplings[ d ][ 1 ], downsamplings[ d ][ 2 ], 1, 1 }; + + final Function levelToName = (level) -> "/" + level; + + // all is 5d now + MultiResolutionLevelInfo[] mrInfoZarr = N5ApiTools.setupMultiResolutionPyramid( + driverVolumeWriter, + levelToName, + dt, + dim, //5d + compression, + blockSize5d, //5d + ds ); // 5d + + final Function levelToMipmapTransform = + (level) -> MipmapTransforms.getMipmapTransformDefault( mrInfoZarr[level].absoluteDownsamplingDouble() ); + + // extract the resolution of the s0 export + // TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied + final VoxelDimensions vx = dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize(); + final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN ); + + IOFunctions.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); + + // create metadata + final OmeNgffMultiScaleMetadata[] meta = OMEZarrAttibutes.createOMEZarrMetadata( + 5, // int n + "/", // String name, I also saw "/" + resolutionS0, // double[] resolutionS0, + "micrometer", //vx.unit() might not be OME-ZARR compatible // String unitXYZ, // e.g micrometer + mrInfoZarr.length, // int numResolutionLevels, + levelToName, + levelToMipmapTransform ); + + // save metadata + + //org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata + // for this to work you need to register an adapter in the N5Factory class + // final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() ); + driverVolumeWriter.setAttribute( "/", "multiscales", meta ); + + // TODO: set extra attributes to load the state + } + else // simple HDF5/N5 export + { + for ( int t = 0; t < numTimepoints; ++t ) + for ( int c = 0; c < numChannels; ++c ) + { + String title = "ch"+c+"tp"+t; + + IOFunctions.println( "Creating 3D " + storageType +" container '" + title + "' in '" + outPathURI + "' ... " ); + + // setup multi-resolution pyramid + final MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupMultiResolutionPyramid( + driverVolumeWriter, + (level) -> title + "/s" + level, + dt, + boundingBox.dimensionsAsLongArray(), + compression, + blockSize, + downsamplings ); + + // TODO: set extra attributes to load the state + } + } + + driverVolumeWriter.close(); + return null; } From 44479cded0111a1628fd6507123668e01d376337 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 20:06:29 -0500 Subject: [PATCH 07/38] directly using mvr --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7aa4ef4..0e6da2f 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 10.6.3 2.3.5 - 5.0.3 + 5.0.4-SNAPSHOT 2.3.1 From ae863173ec3f268f754816aaa32b7b0d243c4870 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 27 Jan 2025 20:06:38 -0500 Subject: [PATCH 08/38] almost finished first version of dataset creation --- .../spark/CreateFusionContainer.java | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 5ecc97c..8faa217 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -25,7 +25,14 @@ import bdv.util.MipmapTransforms; import mpicbg.spim.data.SpimDataException; +import mpicbg.spim.data.sequence.Angle; +import mpicbg.spim.data.sequence.Channel; +import mpicbg.spim.data.sequence.FinalVoxelDimensions; +import mpicbg.spim.data.sequence.Illumination; +import mpicbg.spim.data.sequence.Tile; +import mpicbg.spim.data.sequence.TimePoint; import mpicbg.spim.data.sequence.ViewId; +import mpicbg.spim.data.sequence.ViewSetup; import mpicbg.spim.data.sequence.VoxelDimensions; import net.imglib2.FinalDimensions; import net.imglib2.FinalInterval; @@ -43,6 +50,7 @@ import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; +import net.preibisch.mvrecon.process.n5api.SpimData2Tools; import picocli.CommandLine; import picocli.CommandLine.Option; import util.URITools; @@ -296,6 +304,54 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) // setup datasets and metadata if ( bdv ) { + final long[] bb = boundingBox.dimensionsAsLongArray(); + + final ArrayList< ViewSetup > setups = new ArrayList<>(); + final ArrayList< TimePoint > tps = new ArrayList<>(); + + for ( int t = 0; t < numTimepoints; ++t ) + tps.add( new TimePoint( t ) ); + + // extract the resolution of the s0 export + // TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied + final VoxelDimensions vx = dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize(); + final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN ); + + System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); + + final VoxelDimensions vxNew = new FinalVoxelDimensions( "micrometer", resolutionS0 ); + + for ( int c = 0; c < numChannels; ++c ) + { + setups.add( + new ViewSetup( + c, + "setup " + c, + new FinalDimensions( bb ), + vxNew, + new Tile( 0 ), + new Channel( c, "Channel " + c ), + new Angle( 0 ), + new Illumination( 0 ) ) ); + } + + final SpimData2 dataFusion = + SpimData2Tools.createNewSpimDataForFusion( storageType, outPathURI, xmlOutURI, setups, tps ); + + dataFusion.getSequenceDescription().getViewDescriptions().values().stream().parallel().forEach( vd -> + { + if ( storageType == StorageFormat.N5 ) + { + MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupBdvDatasetsN5( + driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings); + } + else // HDF5 + { + MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupBdvDatasetsHDF5( + driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings); + } + }); + // TODO: set extra attributes to load the state } else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export @@ -313,7 +369,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export final Function levelToName = (level) -> "/" + level; // all is 5d now - MultiResolutionLevelInfo[] mrInfoZarr = N5ApiTools.setupMultiResolutionPyramid( + MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupMultiResolutionPyramid( driverVolumeWriter, levelToName, dt, @@ -323,14 +379,14 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export ds ); // 5d final Function levelToMipmapTransform = - (level) -> MipmapTransforms.getMipmapTransformDefault( mrInfoZarr[level].absoluteDownsamplingDouble() ); + (level) -> MipmapTransforms.getMipmapTransformDefault( mrInfo[level].absoluteDownsamplingDouble() ); // extract the resolution of the s0 export // TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied final VoxelDimensions vx = dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize(); final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN ); - IOFunctions.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); + System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); // create metadata final OmeNgffMultiScaleMetadata[] meta = OMEZarrAttibutes.createOMEZarrMetadata( @@ -338,7 +394,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export "/", // String name, I also saw "/" resolutionS0, // double[] resolutionS0, "micrometer", //vx.unit() might not be OME-ZARR compatible // String unitXYZ, // e.g micrometer - mrInfoZarr.length, // int numResolutionLevels, + mrInfo.length, // int numResolutionLevels, levelToName, levelToMipmapTransform ); @@ -351,7 +407,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export // TODO: set extra attributes to load the state } - else // simple HDF5/N5 export + else // simple (no bdv project) HDF5/N5 export { for ( int t = 0; t < numTimepoints; ++t ) for ( int c = 0; c < numChannels; ++c ) From 149c1dc81bd3044b37f84a8293c8f6977b71f618 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 28 Jan 2025 15:10:19 -0500 Subject: [PATCH 09/38] adding saving of XML --- .../net/preibisch/bigstitcher/spark/CreateFusionContainer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 8faa217..36d075b 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -44,6 +44,7 @@ import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; +import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; import net.preibisch.mvrecon.fiji.spimdata.imgloaders.OMEZarrAttibutes; import net.preibisch.mvrecon.process.export.ExportN5Api; @@ -338,6 +339,8 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) final SpimData2 dataFusion = SpimData2Tools.createNewSpimDataForFusion( storageType, outPathURI, xmlOutURI, setups, tps ); + new XmlIoSpimData2().save( dataFusion, xmlOutPathURI ); + dataFusion.getSequenceDescription().getViewDescriptions().values().stream().parallel().forEach( vd -> { if ( storageType == StorageFormat.N5 ) From 339d81f8a1a1a64dca3d63bbeadea2d10f46c301 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 28 Jan 2025 17:58:38 -0500 Subject: [PATCH 10/38] added metadata storage --- .../spark/CreateFusionContainer.java | 71 +++++++++++++++---- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 36d075b..1135fdf 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -5,6 +5,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; import java.util.function.Function; @@ -31,6 +32,7 @@ import mpicbg.spim.data.sequence.Illumination; import mpicbg.spim.data.sequence.Tile; import mpicbg.spim.data.sequence.TimePoint; +import mpicbg.spim.data.sequence.ViewDescription; import mpicbg.spim.data.sequence.ViewId; import mpicbg.spim.data.sequence.ViewSetup; import mpicbg.spim.data.sequence.VoxelDimensions; @@ -302,9 +304,28 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) } + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/NumTimepoints", numTimepoints ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/NumChannels", numChannels ); + + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/Boundingbox_min", boundingBox.minAsLongArray() ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/Boundingbox_max", boundingBox.maxAsLongArray() ); + + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/PreserveAnisotropy", preserveAnisotropy ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", anisotropyFactor ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/DataType", dt ); + // setup datasets and metadata + MultiResolutionLevelInfo[][] mrInfos; + if ( bdv ) { + System.out.println( "Creating BDV compatible container at '" + outPathURI + "' ... " ); + + if ( storageType == StorageFormat.N5 ) + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" ); + else + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/HDF5" ); + final long[] bb = boundingBox.dimensionsAsLongArray(); final ArrayList< ViewSetup > setups = new ArrayList<>(); @@ -339,18 +360,27 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) final SpimData2 dataFusion = SpimData2Tools.createNewSpimDataForFusion( storageType, outPathURI, xmlOutURI, setups, tps ); - new XmlIoSpimData2().save( dataFusion, xmlOutPathURI ); + new XmlIoSpimData2().save( dataFusion, xmlOutURI ); + + final Collection vds = dataFusion.getSequenceDescription().getViewDescriptions().values(); + + mrInfos = new MultiResolutionLevelInfo[ vds.size() ][]; - dataFusion.getSequenceDescription().getViewDescriptions().values().stream().parallel().forEach( vd -> + vds.stream().parallel().forEach( vd -> { + final int c = vd.getViewSetup().getChannel().getId(); + final int t = vd.getTimePointId(); + if ( storageType == StorageFormat.N5 ) { - MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupBdvDatasetsN5( + mrInfos[ c + t*c ] = N5ApiTools.setupBdvDatasetsN5( driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings); + + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" ); } else // HDF5 { - MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupBdvDatasetsHDF5( + mrInfos[ c + t*c ] = N5ApiTools.setupBdvDatasetsHDF5( driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings); } }); @@ -359,7 +389,9 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) } else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export { - IOFunctions.println( "Creating 5D OME-ZARR metadata for '" + outPathURI + "' ... " ); + System.out.println( "Creating 5D OME-ZARR metadata for '" + outPathURI + "' ... " ); + + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "OME-ZARR" ); final long[] dim3d = boundingBox.dimensionsAsLongArray(); @@ -371,8 +403,10 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export final Function levelToName = (level) -> "/" + level; + mrInfos = new MultiResolutionLevelInfo[ 1 ][]; + // all is 5d now - MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupMultiResolutionPyramid( + mrInfos[ 0 ] = N5ApiTools.setupMultiResolutionPyramid( driverVolumeWriter, levelToName, dt, @@ -382,7 +416,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export ds ); // 5d final Function levelToMipmapTransform = - (level) -> MipmapTransforms.getMipmapTransformDefault( mrInfo[level].absoluteDownsamplingDouble() ); + (level) -> MipmapTransforms.getMipmapTransformDefault( mrInfos[ 0 ][level].absoluteDownsamplingDouble() ); // extract the resolution of the s0 export // TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied @@ -397,7 +431,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export "/", // String name, I also saw "/" resolutionS0, // double[] resolutionS0, "micrometer", //vx.unit() might not be OME-ZARR compatible // String unitXYZ, // e.g micrometer - mrInfo.length, // int numResolutionLevels, + mrInfos[ 0 ].length, // int numResolutionLevels, levelToName, levelToMipmapTransform ); @@ -407,20 +441,25 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export // for this to work you need to register an adapter in the N5Factory class // final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() ); driverVolumeWriter.setAttribute( "/", "multiscales", meta ); - - // TODO: set extra attributes to load the state } else // simple (no bdv project) HDF5/N5 export { - for ( int t = 0; t < numTimepoints; ++t ) - for ( int c = 0; c < numChannels; ++c ) + mrInfos = new MultiResolutionLevelInfo[ numChannels * numTimepoints ][]; + + if ( storageType == StorageFormat.N5 ) + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "N5" ); + else + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "HDF5" ); + + for ( int c = 0; c < numChannels; ++c ) + for ( int t = 0; t < numTimepoints; ++t ) { String title = "ch"+c+"tp"+t; IOFunctions.println( "Creating 3D " + storageType +" container '" + title + "' in '" + outPathURI + "' ... " ); // setup multi-resolution pyramid - final MultiResolutionLevelInfo[] mrInfo = N5ApiTools.setupMultiResolutionPyramid( + mrInfos[ c + t*c ] = N5ApiTools.setupMultiResolutionPyramid( driverVolumeWriter, (level) -> title + "/s" + level, dt, @@ -428,11 +467,13 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export compression, blockSize, downsamplings ); - - // TODO: set extra attributes to load the state } } + + // TODO: set extra attributes to load the state + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", mrInfos ); + driverVolumeWriter.close(); return null; From 36107b745300d1119df8245b13b395b20f3d7029 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 08:57:36 -0500 Subject: [PATCH 11/38] fix bug in array indexing --- .../preibisch/bigstitcher/spark/CreateFusionContainer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 1135fdf..a738525 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -380,7 +380,7 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) } else // HDF5 { - mrInfos[ c + t*c ] = N5ApiTools.setupBdvDatasetsHDF5( + mrInfos[ c + t*numChannels ] = N5ApiTools.setupBdvDatasetsHDF5( driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings); } }); @@ -459,7 +459,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export IOFunctions.println( "Creating 3D " + storageType +" container '" + title + "' in '" + outPathURI + "' ... " ); // setup multi-resolution pyramid - mrInfos[ c + t*c ] = N5ApiTools.setupMultiResolutionPyramid( + mrInfos[ c + t*numChannels ] = N5ApiTools.setupMultiResolutionPyramid( driverVolumeWriter, (level) -> title + "/s" + level, dt, From 04cb2a92333900ae17cf03ef05c4930eee93bac7 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 09:06:01 -0500 Subject: [PATCH 12/38] add more parameters to the JSON --- .../preibisch/bigstitcher/spark/CreateFusionContainer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index a738525..77a5cea 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -64,7 +64,7 @@ public class CreateFusionContainer extends AbstractBasic implements Callable setups = new ArrayList<>(); From fc65dceb43389da9bf6f9bb08d89b08bdcb395f3 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 14:32:51 -0500 Subject: [PATCH 13/38] add one more abstraction layer for cmdline --- .../spark/abstractcmdline/AbstractBasic.java | 12 +++--------- .../AbstractInfrastructure.java | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractInfrastructure.java diff --git a/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractBasic.java b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractBasic.java index 6f50aca..e5bb0bf 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractBasic.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractBasic.java @@ -34,7 +34,7 @@ import picocli.CommandLine.Option; import util.URITools; -public abstract class AbstractBasic implements Callable, Serializable +public abstract class AbstractBasic extends AbstractInfrastructure implements Callable, Serializable { private static final long serialVersionUID = -4916959775650710928L; @@ -44,12 +44,6 @@ public abstract class AbstractBasic implements Callable, Serializable // will be assigned in loadSpimData2() protected URI xmlURI = null; - @Option(names = { "--dryRun" }, description = "perform a 'dry run', i.e. do not save any results (default: false)") - protected boolean dryRun = false; - - @Option(names = "--localSparkBindAddress", description = "specify Spark bind address as localhost") - protected boolean localSparkBindAddress = false; - public SpimData2 loadSpimData2() throws SpimDataException { System.out.println( "xml: " + (xmlURI = URITools.toURI(xmlURIString)) ); @@ -58,7 +52,7 @@ public SpimData2 loadSpimData2() throws SpimDataException return dataGlobal; } - public SpimData2 loadSpimData2( final int numThreads ) throws SpimDataException + public SpimData2 loadSpimData2( final int numFetcherThreads ) throws SpimDataException { final SpimData2 data = loadSpimData2(); @@ -67,7 +61,7 @@ public SpimData2 loadSpimData2( final int numThreads ) throws SpimDataException // set number of fetcher threads (by default set to 0 for spark) final BasicImgLoader imgLoader = sequenceDescription.getImgLoader(); if (imgLoader instanceof ViewerImgLoader) - ((ViewerImgLoader) imgLoader).setNumFetcherThreads( numThreads); + ((ViewerImgLoader) imgLoader).setNumFetcherThreads( numFetcherThreads ); return data; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractInfrastructure.java b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractInfrastructure.java new file mode 100644 index 0000000..54f5da4 --- /dev/null +++ b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractInfrastructure.java @@ -0,0 +1,19 @@ +package net.preibisch.bigstitcher.spark.abstractcmdline; + +import java.io.Serializable; +import java.util.concurrent.Callable; + +import picocli.CommandLine.Option; + +public abstract class AbstractInfrastructure implements Callable, Serializable +{ + + private static final long serialVersionUID = 5199967181629299878L; + + @Option(names = { "--dryRun" }, description = "perform a 'dry run', i.e. do not save any results (default: false)") + protected boolean dryRun = false; + + @Option(names = "--localSparkBindAddress", description = "specify Spark bind address as localhost") + protected boolean localSparkBindAddress = false; + +} From a80776811ed29764113d8210d908201585998f4b Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 14:34:00 -0500 Subject: [PATCH 14/38] start working to change fusion to use created containers --- .../bigstitcher/spark/SparkAffineFusion.java | 104 +++++++++--------- .../bigstitcher/spark/util/N5Util.java | 2 +- .../bigstitcher/spark/util/Spark.java | 10 +- 3 files changed, 60 insertions(+), 56 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 72fbf76..14200ba 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -44,6 +44,7 @@ import net.imglib2.FinalDimensions; import net.imglib2.FinalInterval; import net.imglib2.util.Util; +import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlock; import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlockMasks; @@ -57,6 +58,7 @@ import net.preibisch.mvrecon.process.export.ExportN5Api; import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools; +import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; import net.preibisch.mvrecon.process.n5api.SpimData2Tools; import net.preibisch.mvrecon.process.n5api.SpimData2Tools.InstantiateViewSetup; import picocli.CommandLine; @@ -64,7 +66,7 @@ import util.Grid; import util.URITools; -public class SparkAffineFusion extends AbstractSelectableViews implements Callable, Serializable +public class SparkAffineFusion extends AbstractInfrastructure implements Callable, Serializable { public static enum DataTypeFusion { @@ -74,47 +76,14 @@ public static enum DataTypeFusion private static final long serialVersionUID = -6103761116219617153L; @Option(names = { "-o", "--n5Path" }, required = true, description = "N5/ZARR/HDF5 basse path for saving (must be combined with the option '-d' or '--bdv'), e.g. -o /home/fused.n5 or e.g. s3://myBucket/data.n5") - private String n5PathURIString = null; + private String outputPathURIString = null; - @Option(names = { "-d", "--n5Dataset" }, required = false, description = "Custom N5/ZARR/HDF5 dataset - it must end with '/s0' to be able to compute a multi-resolution pyramid, e.g. -d /ch488/s0") - private String n5Dataset = null; - - @Option(names = { "--bdv" }, required = false, description = "Write a BigDataViewer-compatible dataset specifying TimepointID, ViewSetupId, e.g. --bdv 0,0 or --bdv 4,1") - private String bdvString = null; - - @Option(names = { "-xo", "--xmlout" }, required = false, description = "path to the new BigDataViewer xml project (only valid if --bdv was selected), " - + "e.g. -xo /home/project.xml or -xo s3://myBucket/project.xml (default: dataset.xml in basepath for H5, dataset.xml one directory level above basepath for N5)") - private String xmlOutURIString = null; - - @Option(names = {"-s", "--storage"}, defaultValue = "N5", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, - description = "Dataset storage type, currently supported N5, ZARR (and ONLY for local, multithreaded Spark: HDF5)") + @Option(names = {"-s", "--storage"}, description = "Dataset storage type, can be used to override guessed format (default: guess from file/directory-ending)") private StorageFormat storageType = null; - @Option(names = "--blockSize", description = "blockSize, you can use smaller blocks for HDF5 (default: 128,128,128)") - private String blockSizeString = "128,128,128"; - @Option(names = "--blockScale", description = "how many blocks to use for a single processing step, e.g. 4,4,1 means for blockSize a 128,128,64 that each spark thread writes 512,512,64 (default: 2,2,1)") private String blockScaleString = "2,2,1"; - @Option(names = { "-b", "--boundingBox" }, description = "fuse a specific bounding box listed in the XML (default: fuse everything)") - private String boundingBoxName = null; - - @Option(names = { "--multiRes" }, description = "Automatically create a multi-resolution pyramid (default: false)") - private boolean multiRes = false; - - @Option(names = { "-ds", "--downsampling" }, split = ";", required = false, description = "Manually define steps to create of a multi-resolution pyramid (e.g. -ds 2,2,1; 2,2,1; 2,2,2; 2,2,2)") - private List downsampling = null; - - @Option(names = { "--preserveAnisotropy" }, description = "preserve the anisotropy of the data (default: false)") - private boolean preserveAnisotropy = false; - - @Option(names = { "--anisotropyFactor" }, description = "define the anisotropy factor if preserveAnisotropy is set to true (default: compute from data)") - private double anisotropyFactor = Double.NaN; - - @Option(names = {"-p", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, - description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") - private DataTypeFusion dataTypeFusion = null; - @Option(names = { "--minIntensity" }, description = "min intensity for scaling values to the desired range (required for UINT8 and UINT16), e.g. 0.0") private Double minIntensity = null; @@ -130,11 +99,7 @@ public static enum DataTypeFusion @Option(names = "--maskOffset", description = "allows to make masks larger (+, the mask will include some background) or smaller (-, some fused content will be cut off), warning: in the non-isotropic coordinate space of the raw input images (default: 0.0,0.0,0.0)") private String maskOffset = "0.0,0.0,0.0"; - // null is fine for now (used by multiRes later) - private int[][] downsamplings; - - URI n5PathURI = null, xmlOutURI = null; - + URI outPathURI = null; /** * Prefetching now works with a Executors.newCachedThreadPool(); */ @@ -149,21 +114,54 @@ public Void call() throws Exception System.exit( 0 ); } - if ( (this.n5Dataset == null && this.bdvString == null) || (this.n5Dataset != null && this.bdvString != null) ) + this.outPathURI = URITools.toURI( outputPathURIString ); + System.out.println( "Fused volume: " + outPathURI ); + + if ( storageType == null ) { - System.out.println( "You must define either the n5dataset (e.g. -d /ch488/s0) - OR - the BigDataViewer specification (e.g. --bdv 0,1)"); - return null; - } + if ( outputPathURIString.toLowerCase().endsWith( ".zarr" ) ) + storageType = StorageFormat.ZARR; + else if ( outputPathURIString.toLowerCase().endsWith( ".n5" ) ) + storageType = StorageFormat.N5; + else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toString().toLowerCase().endsWith( ".hdf5" ) ) + storageType = StorageFormat.HDF5; + else + { + System.out.println( "Unable to guess format from URI '" + outPathURI + "', please specify using '-s'"); + return null; + } - if ( this.bdvString != null && xmlOutURIString == null ) + System.out.println( "Guessed format " + storageType + " will be used to open URI '" + outPathURI + "', you can override it using '-s'"); + } + else { - System.out.println( "Please specify the output XML for the BDV dataset: -xo"); - return null; + System.out.println( "Format " + storageType + " will be used to open " + outPathURI ); } - Import.validateInputParameters( dataTypeFusion, minIntensity, maxIntensity); + final N5Writer driverVolumeWriter = N5Util.createN5Writer( outPathURI, storageType ); + + final String fusionFormat = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/FusionFormat", String.class ); - final SpimData2 dataGlobal = this.loadSpimData2(); + final boolean bdv = fusionFormat.toLowerCase().contains( "BDV" ); + + final URI xmlURI = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/InputXML", URI.class ); + final int numTimepoints = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumTimepoints", int.class ); + final int numChannels = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumChannels", int.class ); + + final long[] bbMin = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_min", long[].class ); + final long[] bbMax = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_max", long[].class ); + + final boolean preserveAnisotropy = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/PreserveAnisotropy", boolean.class ); + final double anisotropyFactor = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", double.class ); + final DataType dataType = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/DataType", DataType.class ); + + System.out.println( "Input XML: " + xmlURI ); + + final MultiResolutionLevelInfo[][] mrInfos = + driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", MultiResolutionLevelInfo[][].class ); + + /* + final SpimData2 dataGlobal = Spark.getJobSpimData2( loadFromContainer, 0 ); if ( dataGlobal == null ) return null; @@ -173,10 +171,8 @@ public Void call() throws Exception if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) return null; - BoundingBox boundingBox = Import.getBoundingBox( dataGlobal, viewIdsGlobal, boundingBoxName ); + BoundingBox boundingBox = loadFromContainer; - this.n5PathURI = URITools.toURI( n5PathURIString ); - System.out.println( "Fused volume: " + n5PathURI ); if ( this.bdvString != null ) { @@ -412,7 +408,7 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset + "'" ); System.out.println( "done, took: " + (System.currentTimeMillis() - time ) + " ms." ); - + */ return null; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/N5Util.java b/src/main/java/net/preibisch/bigstitcher/spark/util/N5Util.java index be2a3a4..51c4786 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/N5Util.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/N5Util.java @@ -63,7 +63,7 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) } catch ( Exception e ) { - IOFunctions.println( "Couldn't create " + storageType + " container '" + n5PathURI + "': " + e ); + IOFunctions.println( "Couldn't create/open " + storageType + " container '" + n5PathURI + "': " + e ); return null; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java index ac6ee22..c645ff4 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java @@ -226,6 +226,14 @@ public static String getSparkExecutorId() { * @return a new data instance optimized for use within single-threaded Spark tasks. */ public static SpimData2 getSparkJobSpimData2( final URI xmlPath ) throws SpimDataException + { + return getJobSpimData2( xmlPath, 0 ); + } + + /** + * @return a new data instance optimized for multi-threaded tasks. + */ + public static SpimData2 getJobSpimData2( final URI xmlPath, final int numFetcherThreads ) throws SpimDataException { final SpimData2 data = new XmlIoSpimData2().load(xmlPath); final SequenceDescription sequenceDescription = data.getSequenceDescription(); @@ -233,7 +241,7 @@ public static SpimData2 getSparkJobSpimData2( final URI xmlPath ) throws SpimDat // set number of fetcher threads to 0 for spark usage final BasicImgLoader imgLoader = sequenceDescription.getImgLoader(); if (imgLoader instanceof ViewerImgLoader) { - ((ViewerImgLoader) imgLoader).setNumFetcherThreads(0); + ((ViewerImgLoader) imgLoader).setNumFetcherThreads( numFetcherThreads ); } LOG.info("getSparkJobSpimData2: loaded {}, xmlPath={} on executorId={}", From ce0c82bec0996b2e09bba7990d58fbf94f437c64 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 14:50:20 -0500 Subject: [PATCH 15/38] transferring more metadata to fusion --- .../spark/CreateFusionContainer.java | 2 +- .../bigstitcher/spark/SparkAffineFusion.java | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 77a5cea..70ead55 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -427,7 +427,7 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export final VoxelDimensions vx = dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize(); final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN ); - System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); + System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "micrometer" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() ); // create metadata final OmeNgffMultiScaleMetadata[] meta = OMEZarrAttibutes.createOMEZarrMetadata( diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 14200ba..b4ce834 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -150,28 +150,41 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt final long[] bbMin = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_min", long[].class ); final long[] bbMax = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_max", long[].class ); + + final BoundingBox bb = new BoundingBox( new FinalInterval( bbMin, bbMax ) ); final boolean preserveAnisotropy = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/PreserveAnisotropy", boolean.class ); final double anisotropyFactor = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", double.class ); final DataType dataType = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/DataType", DataType.class ); + System.out.println( "FusionFormat: " + fusionFormat ); System.out.println( "Input XML: " + xmlURI ); + System.out.println( "BDV project: " + bdv ); + System.out.println( "numTimepoints of fused dataset(s): " + numTimepoints ); + System.out.println( "numChannels of fused dataset(s): " + numChannels ); + System.out.println( "BoundingBox: " + bb ); + System.out.println( "preserveAnisotropy: " + preserveAnisotropy ); + System.out.println( "anisotropyFactor: " + anisotropyFactor ); + System.out.println( "dataType: " + dataType ); final MultiResolutionLevelInfo[][] mrInfos = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", MultiResolutionLevelInfo[][].class ); - /* - final SpimData2 dataGlobal = Spark.getJobSpimData2( loadFromContainer, 0 ); + System.out.println( "Loaded " + mrInfos.length + " metadata object for fused " + storageType + " volume(s)" ); + + final SpimData2 dataGlobal = Spark.getJobSpimData2( xmlURI, 0 ); if ( dataGlobal == null ) return null; + /* + final ArrayList< ViewId > viewIdsGlobal = this.loadViewIds( dataGlobal ); if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) return null; - BoundingBox boundingBox = loadFromContainer; + BoundingBox boundingBox = loadFromContainer; if ( this.bdvString != null ) From 04368d21dc0e82c0524976f7aaaeff9f27c2d43e Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Wed, 29 Jan 2025 19:43:26 -0500 Subject: [PATCH 16/38] add min & max intensity to fusion definition --- .../spark/CreateFusionContainer.java | 27 +++++++++++++++++++ .../bigstitcher/spark/SparkAffineFusion.java | 27 ++++++++++++------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 70ead55..d3d5c08 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -91,6 +91,12 @@ public static enum Compressions { Lz4, Gzip, Zstandard, Blosc, Bzip2, Xz, Raw }; description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") private DataTypeFusion dataTypeFusion = null; + @Option(names = { "--minIntensity" }, description = "optionally adjust min intensity for scaling values to the desired range for UINT8 and UINT16 output, (default: 0)") + private Double minIntensity = null; + + @Option(names = { "--maxIntensity" }, description = "optionally adjust max intensity for scaling values to the desired range for UINT8 and UINT16 output, (default: 255 for UINT8, 65535 for UINT16)") + private Double maxIntensity = null; + @Option(names = { "--bdv" }, required = false, description = "Write a BigDataViewer-compatible dataset (default: false)") private boolean bdv = false; @@ -240,15 +246,30 @@ else if ( this.compression == Compressions.Raw ) System.out.println( "Compression level: " + ( compressionLevel == null ? "default" : compressionLevel ) ); final DataType dt; + Double minIntensity, maxIntensity; if ( dataTypeFusion == DataTypeFusion.UINT8 ) + { dt = DataType.UINT8; + minIntensity = (this.minIntensity == null) ? 0 : this.minIntensity; + maxIntensity = (this.maxIntensity == null) ? 255 : this.maxIntensity; + } else if ( dataTypeFusion == DataTypeFusion.UINT16 ) + { dt = DataType.UINT16; + minIntensity = (this.minIntensity == null) ? 0 : this.minIntensity; + maxIntensity = (this.maxIntensity == null) ? 65535 : this.maxIntensity; + } else if ( dataTypeFusion == DataTypeFusion.FLOAT32 ) + { dt = DataType.FLOAT32; + minIntensity = maxIntensity = null; + } else + { dt = null; + minIntensity = maxIntensity = null; + } System.out.println( "Data type: " + dt ); @@ -316,6 +337,12 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", anisotropyFactor ); driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/DataType", dt ); + if ( minIntensity != null && maxIntensity != null ) + { + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/MinIntensity", minIntensity ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/MaxIntensity", maxIntensity ); + } + // setup datasets and metadata MultiResolutionLevelInfo[][] mrInfos; diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index b4ce834..70e218b 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -84,20 +84,15 @@ public static enum DataTypeFusion @Option(names = "--blockScale", description = "how many blocks to use for a single processing step, e.g. 4,4,1 means for blockSize a 128,128,64 that each spark thread writes 512,512,64 (default: 2,2,1)") private String blockScaleString = "2,2,1"; - @Option(names = { "--minIntensity" }, description = "min intensity for scaling values to the desired range (required for UINT8 and UINT16), e.g. 0.0") - private Double minIntensity = null; + //@Option(names = { "--masks" }, description = "save only the masks (this will not fuse the images)") + //private boolean masks = false; - @Option(names = { "--maxIntensity" }, description = "max intensity for scaling values to the desired range (required for UINT8 and UINT16), e.g. 2048.0") - private Double maxIntensity = null; - - @Option(names = { "--masks" }, description = "save only the masks (this will not fuse the images)") - private boolean masks = false; + //@Option(names = "--maskOffset", description = "allows to make masks larger (+, the mask will include some background) or smaller (-, some fused content will be cut off), warning: in the non-isotropic coordinate space of the raw input images (default: 0.0,0.0,0.0)") + //private String maskOffset = "0.0,0.0,0.0"; @Option(names = { "--firstTileWins" }, description = "use firstTileWins fusion strategy (default: false - using weighted average blending fusion)") private boolean firstTileWins = false; - @Option(names = "--maskOffset", description = "allows to make masks larger (+, the mask will include some background) or smaller (-, some fused content will be cut off), warning: in the non-isotropic coordinate space of the raw input images (default: 0.0,0.0,0.0)") - private String maskOffset = "0.0,0.0,0.0"; URI outPathURI = null; /** @@ -157,6 +152,18 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt final double anisotropyFactor = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", double.class ); final DataType dataType = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/DataType", DataType.class ); + final double minIntensity, maxIntensity; + + if ( driverVolumeWriter.listAttributes( "Bigstitcher-Spark").containsKey( "MinIntensity" ) ) + { + minIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MinIntensity", double.class ); + maxIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MaxIntensity", double.class ); + } + else + { + minIntensity = maxIntensity = Double.NaN; + } + System.out.println( "FusionFormat: " + fusionFormat ); System.out.println( "Input XML: " + xmlURI ); System.out.println( "BDV project: " + bdv ); @@ -166,6 +173,8 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt System.out.println( "preserveAnisotropy: " + preserveAnisotropy ); System.out.println( "anisotropyFactor: " + anisotropyFactor ); System.out.println( "dataType: " + dataType ); + System.out.println( "minIntensity: " + minIntensity ); + System.out.println( "maxIntensity: " + maxIntensity ); final MultiResolutionLevelInfo[][] mrInfos = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", MultiResolutionLevelInfo[][].class ); From 7a82298e2c3de32ad442c8cc7e45c4deb3c71554 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Thu, 30 Jan 2025 15:47:58 -0500 Subject: [PATCH 17/38] getting closer, first prototype of fusion --- .../spark/CreateFusionContainer.java | 1 + .../bigstitcher/spark/SparkAffineFusion.java | 396 +++++++++++------- .../spark/fusion/WriteSuperBlock.java | 3 - 3 files changed, 236 insertions(+), 164 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index d3d5c08..551fb5a 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -336,6 +336,7 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR ) driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/PreserveAnisotropy", preserveAnisotropy ); driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", anisotropyFactor ); driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/DataType", dt ); + driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/BlockSize", blockSize ); if ( minIntensity != null && maxIntensity != null ) { diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 70e218b..302b925 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -26,6 +26,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; @@ -36,14 +37,32 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import mpicbg.spim.data.SpimDataException; +import mpicbg.spim.data.sequence.SequenceDescription; +import mpicbg.spim.data.sequence.ViewDescription; import mpicbg.spim.data.sequence.ViewId; import net.imglib2.FinalDimensions; import net.imglib2.FinalInterval; +import net.imglib2.Interval; +import net.imglib2.RandomAccessible; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.converter.Converter; +import net.imglib2.converter.RealUnsignedByteConverter; +import net.imglib2.converter.RealUnsignedShortConverter; +import net.imglib2.realtransform.AffineTransform3D; +import net.imglib2.type.NativeType; +import net.imglib2.type.Type; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Intervals; import net.imglib2.util.Util; +import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlock; @@ -53,9 +72,13 @@ import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.bigstitcher.spark.util.N5Util; import net.preibisch.bigstitcher.spark.util.Spark; +import net.preibisch.mvrecon.fiji.plugin.fusion.FusionGUI.FusionType; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; import net.preibisch.mvrecon.process.export.ExportN5Api; +import net.preibisch.mvrecon.process.fusion.blk.BlkAffineFusion; +import net.preibisch.mvrecon.process.fusion.transformed.FusedRandomAccessibleInterval.Fusion; +import net.preibisch.mvrecon.process.fusion.transformed.TransformVirtual; import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; @@ -146,36 +169,40 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt final long[] bbMin = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_min", long[].class ); final long[] bbMax = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_max", long[].class ); - final BoundingBox bb = new BoundingBox( new FinalInterval( bbMin, bbMax ) ); + final BoundingBox boundingBox = new BoundingBox( new FinalInterval( bbMin, bbMax ) ); final boolean preserveAnisotropy = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/PreserveAnisotropy", boolean.class ); final double anisotropyFactor = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/AnisotropyFactor", double.class ); + final int[] blockSize = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/BlockSize", int[].class ); + final DataType dataType = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/DataType", DataType.class ); + System.out.println( "FusionFormat: " + fusionFormat ); + System.out.println( "Input XML: " + xmlURI ); + System.out.println( "BDV project: " + bdv ); + System.out.println( "numTimepoints of fused dataset(s): " + numTimepoints ); + System.out.println( "numChannels of fused dataset(s): " + numChannels ); + System.out.println( "BoundingBox: " + boundingBox ); + System.out.println( "preserveAnisotropy: " + preserveAnisotropy ); + System.out.println( "anisotropyFactor: " + anisotropyFactor ); + System.out.println( "blockSize: " + Arrays.toString( blockSize ) ); + System.out.println( "dataType: " + dataType ); + final double minIntensity, maxIntensity; if ( driverVolumeWriter.listAttributes( "Bigstitcher-Spark").containsKey( "MinIntensity" ) ) { minIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MinIntensity", double.class ); maxIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MaxIntensity", double.class ); + + System.out.println( "minIntensity: " + minIntensity ); + System.out.println( "maxIntensity: " + maxIntensity ); } else { minIntensity = maxIntensity = Double.NaN; } - System.out.println( "FusionFormat: " + fusionFormat ); - System.out.println( "Input XML: " + xmlURI ); - System.out.println( "BDV project: " + bdv ); - System.out.println( "numTimepoints of fused dataset(s): " + numTimepoints ); - System.out.println( "numChannels of fused dataset(s): " + numChannels ); - System.out.println( "BoundingBox: " + bb ); - System.out.println( "preserveAnisotropy: " + preserveAnisotropy ); - System.out.println( "anisotropyFactor: " + anisotropyFactor ); - System.out.println( "dataType: " + dataType ); - System.out.println( "minIntensity: " + minIntensity ); - System.out.println( "maxIntensity: " + maxIntensity ); - final MultiResolutionLevelInfo[][] mrInfos = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", MultiResolutionLevelInfo[][].class ); @@ -186,48 +213,47 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt if ( dataGlobal == null ) return null; - /* + final ArrayList< ViewId > viewIdsGlobal; - final ArrayList< ViewId > viewIdsGlobal = this.loadViewIds( dataGlobal ); + if ( + dataGlobal.getSequenceDescription().getAllChannelsOrdered().size() != numChannels || + dataGlobal.getSequenceDescription().getTimePoints().getTimePointsOrdered().size() != numTimepoints ) + { + System.out.println( + "The number of channels and timepoint in XML does not match the number in the export dataset.\n" + + "You have to specify which ViewIds/Channels/Illuminations/Tiles/Angles/Timepoints should be fused into\n" + + "a specific 3D volume in the fusion dataset:\n"); + // TODO: support that + /* + final ArrayList< ViewId > viewIdsGlobal = this.loadViewIds( dataGlobal ); + + if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) + return null; + */ - if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) return null; - - BoundingBox boundingBox = loadFromContainer; - - - if ( this.bdvString != null ) + } + else { - this.xmlOutURI = URITools.toURI( xmlOutURIString ); - System.out.println( "XML: " + xmlOutURI ); + viewIdsGlobal = Import.getViewIds( dataGlobal ); } - final int[] blockSize = Import.csvStringToIntArray(blockSizeString); final int[] blocksPerJob = Import.csvStringToIntArray(blockScaleString); System.out.println( "Fusing: " + boundingBox.getTitle() + ": " + Util.printInterval( boundingBox ) + " with blocksize " + Util.printCoordinates( blockSize ) + " and " + Util.printCoordinates( blocksPerJob ) + " blocks per job" ); - final DataType dataType; - - if ( dataTypeFusion == DataTypeFusion.UINT8 ) - { + if ( dataType == DataType.UINT8 ) System.out.println( "Fusing to UINT8, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); - dataType = DataType.UINT8; - } - else if ( dataTypeFusion == DataTypeFusion.UINT16 ) - { + else if ( dataType == DataType.UINT16 ) System.out.println( "Fusing to UINT16, min intensity = " + minIntensity + ", max intensity = " + maxIntensity ); - dataType = DataType.UINT16; - } else - { System.out.println( "Fusing to FLOAT32" ); - dataType = DataType.FLOAT32; - } + /* final double[] maskOff = Import.csvStringToDoubleArray(maskOffset); if ( masks ) System.out.println( "Fusing ONLY MASKS! Mask offset: " + Util.printCoordinates( maskOff ) ); + */ // // final variables for Spark @@ -235,62 +261,9 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) final long[] minBB = boundingBox.minAsLongArray(); final long[] maxBB = boundingBox.maxAsLongArray(); - if ( preserveAnisotropy ) - { - System.out.println( "Preserving anisotropy."); - - if ( Double.isNaN( anisotropyFactor ) ) - { - anisotropyFactor = TransformationTools.getAverageAnisotropyFactor( dataGlobal, viewIdsGlobal ); - - System.out.println( "Anisotropy factor [computed from data]: " + anisotropyFactor ); - } - else - { - System.out.println( "Anisotropy factor [provided]: " + anisotropyFactor ); - } - - // prepare downsampled boundingbox - minBB[ 2 ] = Math.round( Math.floor( minBB[ 2 ] / anisotropyFactor ) ); - maxBB[ 2 ] = Math.round( Math.ceil( maxBB[ 2 ] / anisotropyFactor ) ); - - boundingBox = new BoundingBox( new FinalInterval(minBB, maxBB) ); - - System.out.println( "Adjusted bounding box (anisotropy preserved: " + Util.printInterval( boundingBox ) ); - } - - // - // set up downsampling (if wanted) - // - if ( !Downsampling.testDownsamplingParameters( this.multiRes, this.downsampling, this.n5Dataset ) ) - return null; - - if ( multiRes ) - downsamplings = ExportN5Api.estimateMultiResPyramid( new FinalDimensions( boundingBox ), anisotropyFactor ); - else if ( this.downsampling != null ) - downsamplings = Import.csvStringListToDownsampling( this.downsampling ); - else - downsamplings = null;// for new code: new int[][]{{ 1, 1, 1 }}; - final long[] dimensions = boundingBox.dimensionsAsLongArray(); - final String n5Dataset = this.n5Dataset != null ? this.n5Dataset : N5ApiTools.createBDVPath( this.bdvString, 0, this.storageType ); - - // TODO: expose - final Compression compression = new ZstandardCompression( 3 );// new GzipCompression( 1 ); - - final double minIntensity = ( dataTypeFusion != DataTypeFusion.FLOAT32 ) ? this.minIntensity : 0; - final double range; - if ( dataTypeFusion == DataTypeFusion.UINT8 ) - range = ( this.maxIntensity - this.minIntensity ) / 255.0; - else if ( dataTypeFusion == DataTypeFusion.UINT16 ) - range = ( this.maxIntensity - this.minIntensity ) / 65535.0; - else - range = 0; - - // TODO: improve (e.g. make ViewId serializable) - final int[][] serializedViewIds = Spark.serializeViewIds(viewIdsGlobal); - + // TODO: do we still need this? try { // trigger the N5-blosc error, because if it is triggered for the first @@ -299,6 +272,14 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) } catch (Exception e ) {} + /* + final String n5Dataset = this.n5Dataset != null ? this.n5Dataset : N5ApiTools.createBDVPath( this.bdvString, 0, this.storageType ); + + // TODO: expose + final Compression compression = new ZstandardCompression( 3 );// new GzipCompression( 1 ); + + final double minIntensity = ( dataTypeFusion != DataTypeFusion.FLOAT32 ) ? this.minIntensity : 0; + System.out.println( "Format being written: " + storageType ); final N5Writer driverVolumeWriter = N5Util.createN5Writer( n5PathURI, storageType ); @@ -309,17 +290,6 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) dataType, compression ); - // using bigger blocksizes than being stored for efficiency (needed for very large datasets) - final int[] superBlockSize = new int[ 3 ]; - Arrays.setAll( superBlockSize, d -> blockSize[ d ] * blocksPerJob[ d ] ); - final List grid = Grid.create(dimensions, - superBlockSize, - blockSize); - - System.out.println( "numJobs = " + grid.size() ); - - driverVolumeWriter.setAttribute( n5Dataset, "offset", minBB ); - // saving metadata if it is bdv-compatible (we do this first since it might fail) if ( bdvString != null ) { @@ -357,6 +327,7 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) System.out.println( "Done writing BDV metadata."); } + */ final SparkConf conf = new SparkConf().setAppName("AffineFusion"); @@ -366,71 +337,174 @@ else if ( dataTypeFusion == DataTypeFusion.UINT16 ) final JavaSparkContext sc = new JavaSparkContext(conf); sc.setLogLevel("ERROR"); - final JavaRDD rdd = sc.parallelize( grid ); + final SequenceDescription sd = dataGlobal.getSequenceDescription(); - final long time = System.currentTimeMillis(); + final HashMap< Integer, Integer > tpIdToTpIndex = new HashMap<>(); + final HashMap< Integer, Integer > chIdToChIndex = new HashMap<>(); - if ( masks ) - rdd.foreach( new WriteSuperBlockMasks( - xmlURI, - preserveAnisotropy, - anisotropyFactor, - minBB, - n5PathURI, - n5Dataset, - storageType, - serializedViewIds, - dataTypeFusion == DataTypeFusion.UINT8, - dataTypeFusion == DataTypeFusion.UINT16, - maskOff, - blockSize ) ); - else - rdd.foreach( new WriteSuperBlock( - xmlURI, - preserveAnisotropy, - anisotropyFactor, - minBB, - n5PathURI, - n5Dataset, - bdvString, - storageType, - serializedViewIds, - dataTypeFusion == DataTypeFusion.UINT8, - dataTypeFusion == DataTypeFusion.UINT16, - minIntensity, - range, - blockSize, - firstTileWins ) ); - - if ( this.downsamplings != null ) - { - // TODO: run common downsampling code (affine, non-rigid, downsampling-only) - Downsampling.createDownsampling( - n5PathURI, - n5Dataset, - driverVolumeWriter, - dimensions, - storageType, - blockSize, - dataType, - compression, - downsamplings, - bdvString != null, - sc ); - } + for ( int t = 0; t < sd.getTimePoints().getTimePointsOrdered().size(); ++t ) + tpIdToTpIndex.put( sd.getTimePoints().getTimePointsOrdered().get( t ).getId(), t ); - sc.close(); + for ( int c = 0; c < sd.getAllChannelsOrdered().size(); ++c ) + chIdToChIndex.put( sd.getAllChannelsOrdered().get( c ).getId(), c ); - // close main writer (is shared over Spark-threads if it's HDF5, thus just closing it here) - driverVolumeWriter.close(); + for ( int c = 0; c < numChannels; ++c ) + for ( int t = 0; t < numTimepoints; ++t ) + { + System.out.println( "\nProcessing channel " + c + ", timepoint " + t ); + System.out.println( "\n-----------------------------------" ); + + final int tIndex = t; + final int cIndex = c; + + final ArrayList< ViewId > viewIds = new ArrayList<>(); + viewIdsGlobal.forEach( viewId -> { + final ViewDescription vd = sd.getViewDescription( viewId ); + + if ( tpIdToTpIndex.get( vd.getTimePointId() ) == tIndex && chIdToChIndex.get( vd.getViewSetup().getChannel().getId() ) == cIndex ) + viewIds.add( viewId ); + }); + + System.out.println( "Fusing " + viewIds.size() + " views for this 3D volume ... " ); + + final MultiResolutionLevelInfo[] mrInfo = mrInfos[ c + t*numChannels ]; + + // using bigger blocksizes than being stored for efficiency (needed for very large datasets) + final int[] superBlockSize = new int[ 3 ]; + Arrays.setAll( superBlockSize, d -> blockSize[ d ] * blocksPerJob[ d ] ); + final List grid = Grid.create(dimensions, + superBlockSize, + blockSize); + + System.out.println( "numJobs = " + grid.size() ); + + //driverVolumeWriter.setAttribute( n5Dataset, "offset", minBB ); + + final JavaRDD rdd = sc.parallelize( grid ); + + final long time = System.currentTimeMillis(); + + //TODO: prefetchExecutor!! + + rdd.foreach( + gridBlock -> + { + final SpimData2 dataLocal = Spark.getSparkJobSpimData2(xmlURI); + + final HashMap< ViewId, AffineTransform3D > registrations = + TransformVirtual.adjustAllTransforms( + viewIds, + dataLocal.getViewRegistrations().getViewRegistrations(), + anisotropyFactor, + Double.NaN ); + + final Converter conv; + final Type type; + + if ( dataType == DataType.UINT8 ) + { + conv = new RealUnsignedByteConverter<>( minIntensity, maxIntensity ); + type = new UnsignedByteType(); + } + else if ( dataType == DataType.UINT16 ) + { + conv = new RealUnsignedShortConverter<>( minIntensity, maxIntensity ); + type = new UnsignedShortType(); + } + else + { + conv = null; + type = new FloatType(); + } + + RandomAccessibleInterval img = BlkAffineFusion.init( + conv, + dataLocal.getSequenceDescription().getImgLoader(), + viewIds, + registrations, + dataLocal.getSequenceDescription().getViewDescriptions(), + firstTileWins ? FusionType.FIRST : FusionType.AVG_BLEND,//fusion.getFusionType(), + 1, // linear interpolation + null, // intensity correction + boundingBox, + (RealType & NativeType)type, + blockSize ); + + final long[] blockOffset, blockSizeExport, gridOffset; + + final RandomAccessible image; + + // 5D OME-ZARR CONTAINER + if ( storageType == StorageFormat.ZARR ) + { + // gridBlock is 3d, make it 5d + blockOffset = new long[] { gridBlock[0][0], gridBlock[0][1], gridBlock[0][2], cIndex, tIndex }; + blockSizeExport = new long[] { gridBlock[1][0], gridBlock[1][1], gridBlock[1][2], 1, 1 }; + gridOffset = new long[] { gridBlock[2][0], gridBlock[2][1], gridBlock[2][2], cIndex, tIndex }; // because blocksize in C & T is 1 + + // img is 3d, make it 5d + // the same information is returned no matter which index is queried in C and T + image = Views.addDimension( Views.addDimension( img ) ); + } + else + { + blockOffset = gridBlock[0]; + blockSizeExport = gridBlock[1]; + gridOffset = gridBlock[2]; + + image = img; + } + + final Interval block = + Intervals.translate( + new FinalInterval( blockSizeExport ), + blockOffset ); + + final RandomAccessibleInterval source = + Views.interval( image, block ); + + final RandomAccessibleInterval sourceGridBlock = + Views.offsetInterval(source, blockOffset, blockSizeExport); + + final N5Writer driverVolumeWriterLocal = N5Util.createN5Writer( outPathURI, storageType ); + + N5Utils.saveBlock(sourceGridBlock, driverVolumeWriterLocal, mrInfo[ 0 ].dataset, gridOffset ); + + if ( N5Util.sharedHDF5Writer == null ) + driverVolumeWriterLocal.close(); + } ); + /* + if ( this.downsamplings != null ) + { + // TODO: run common downsampling code (affine, non-rigid, downsampling-only) + Downsampling.createDownsampling( + n5PathURI, + n5Dataset, + driverVolumeWriter, + dimensions, + storageType, + blockSize, + dataType, + compression, + downsamplings, + bdvString != null, + sc ); + } - if ( multiRes ) - System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset.substring( 0, n5Dataset.length() - 3) + "'" ); - else - System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset + "'" ); + // close main writer (is shared over Spark-threads if it's HDF5, thus just closing it here) + driverVolumeWriter.close(); + + if ( multiRes ) + System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset.substring( 0, n5Dataset.length() - 3) + "'" ); + else + System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset + "'" ); + + System.out.println( "done, took: " + (System.currentTimeMillis() - time ) + " ms." ); + */ + } + + sc.close(); - System.out.println( "done, took: " + (System.currentTimeMillis() - time ) + " ms." ); - */ return null; } diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java index df7f810..fa76a62 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java @@ -72,8 +72,6 @@ public class WriteSuperBlock implements VoidFunction< long[][] > private final String n5Dataset; - private final String bdvString; - private final StorageFormat storageType; private final int[][] serializedViewIds; @@ -113,7 +111,6 @@ public WriteSuperBlock( this.minBB = minBB; this.n5PathURI = n5PathURI; this.n5Dataset = n5Dataset; - this.bdvString = bdvString; this.storageType = storageType; this.serializedViewIds = serializedViewIds; this.uint8 = uint8; From f210d6a4b4728330d7f319ecdc0dd50fd73c7741 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Thu, 30 Jan 2025 16:09:31 -0500 Subject: [PATCH 18/38] fix a few Spark-related bugs --- .../bigstitcher/spark/SparkAffineFusion.java | 17 +++++++++-------- .../preibisch/bigstitcher/spark/util/Spark.java | 3 +-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 302b925..3d23914 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -258,9 +258,6 @@ else if ( dataType == DataType.UINT16 ) // // final variables for Spark // - final long[] minBB = boundingBox.minAsLongArray(); - final long[] maxBB = boundingBox.maxAsLongArray(); - final long[] dimensions = boundingBox.dimensionsAsLongArray(); // TODO: do we still need this? @@ -352,7 +349,7 @@ else if ( dataType == DataType.UINT16 ) for ( int t = 0; t < numTimepoints; ++t ) { System.out.println( "\nProcessing channel " + c + ", timepoint " + t ); - System.out.println( "\n-----------------------------------" ); + System.out.println( "-----------------------------------" ); final int tIndex = t; final int cIndex = c; @@ -362,12 +359,17 @@ else if ( dataType == DataType.UINT16 ) final ViewDescription vd = sd.getViewDescription( viewId ); if ( tpIdToTpIndex.get( vd.getTimePointId() ) == tIndex && chIdToChIndex.get( vd.getViewSetup().getChannel().getId() ) == cIndex ) - viewIds.add( viewId ); + viewIds.add( new ViewId( viewId.getTimePointId(), viewId.getViewSetupId() ) ); }); System.out.println( "Fusing " + viewIds.size() + " views for this 3D volume ... " ); - final MultiResolutionLevelInfo[] mrInfo = mrInfos[ c + t*numChannels ]; + final MultiResolutionLevelInfo[] mrInfo; + + if ( storageType == StorageFormat.ZARR ) + mrInfo = mrInfos[ 0 ]; + else + mrInfo = mrInfos[ c + t*numChannels ]; // using bigger blocksizes than being stored for efficiency (needed for very large datasets) final int[] superBlockSize = new int[ 3 ]; @@ -385,7 +387,6 @@ else if ( dataType == DataType.UINT16 ) final long time = System.currentTimeMillis(); //TODO: prefetchExecutor!! - rdd.foreach( gridBlock -> { @@ -426,7 +427,7 @@ else if ( dataType == DataType.UINT16 ) firstTileWins ? FusionType.FIRST : FusionType.AVG_BLEND,//fusion.getFusionType(), 1, // linear interpolation null, // intensity correction - boundingBox, + new BoundingBox( new FinalInterval( bbMin, bbMax ) ), (RealType & NativeType)type, blockSize ); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java b/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java index c645ff4..effccba 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/Spark.java @@ -244,8 +244,7 @@ public static SpimData2 getJobSpimData2( final URI xmlPath, final int numFetcher ((ViewerImgLoader) imgLoader).setNumFetcherThreads( numFetcherThreads ); } - LOG.info("getSparkJobSpimData2: loaded {}, xmlPath={} on executorId={}", - data, xmlPath, getSparkExecutorId()); + //LOG.info("getSparkJobSpimData2: loaded {}, xmlPath={} on executorId={}", data, xmlPath, getSparkExecutorId()); return data; } From 08513c704d8e33fa9a47117384c1630341743805 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Thu, 30 Jan 2025 16:33:39 -0500 Subject: [PATCH 19/38] fix more little bugs --- .../spark/CreateFusionContainer.java | 2 +- .../bigstitcher/spark/SparkAffineFusion.java | 93 +++++++++++++------ 2 files changed, 68 insertions(+), 27 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java index 551fb5a..9a9d612 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java @@ -87,7 +87,7 @@ public static enum Compressions { Lz4, Gzip, Zstandard, Blosc, Bzip2, Xz, Raw }; @Option(names = "--blockSize", description = "blockSize (default: 128,128,128)") private String blockSizeString = "128,128,128"; - @Option(names = {"-p", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + @Option(names = {"-d", "--dataType"}, defaultValue = "FLOAT32", showDefaultValue = CommandLine.Help.Visibility.ALWAYS, description = "Data type, UINT8 [0...255], UINT16 [0...65535] and FLOAT32 are supported, when choosing UINT8 or UINT16 you must define min and max intensity (default: FLOAT32)") private DataTypeFusion dataTypeFusion = null; diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 3d23914..756ad64 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -26,9 +26,11 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -41,6 +43,7 @@ import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.janelia.scicomp.n5.zstandard.ZstandardCompression; +import ij.IJ; import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.sequence.SequenceDescription; import mpicbg.spim.data.sequence.ViewDescription; @@ -72,6 +75,7 @@ import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.bigstitcher.spark.util.N5Util; import net.preibisch.bigstitcher.spark.util.Spark; +import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.plugin.fusion.FusionGUI.FusionType; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; @@ -345,6 +349,8 @@ else if ( dataType == DataType.UINT16 ) for ( int c = 0; c < sd.getAllChannelsOrdered().size(); ++c ) chIdToChIndex.put( sd.getAllChannelsOrdered().get( c ).getId(), c ); + final long totalTime = System.currentTimeMillis(); + for ( int c = 0; c < numChannels; ++c ) for ( int t = 0; t < numTimepoints; ++t ) { @@ -384,7 +390,7 @@ else if ( dataType == DataType.UINT16 ) final JavaRDD rdd = sc.parallelize( grid ); - final long time = System.currentTimeMillis(); + long time = System.currentTimeMillis(); //TODO: prefetchExecutor!! rdd.foreach( @@ -418,7 +424,7 @@ else if ( dataType == DataType.UINT16 ) type = new FloatType(); } - RandomAccessibleInterval img = BlkAffineFusion.init( + final RandomAccessibleInterval img = BlkAffineFusion.init( conv, dataLocal.getSequenceDescription().getImgLoader(), viewIds, @@ -474,36 +480,71 @@ else if ( dataType == DataType.UINT16 ) if ( N5Util.sharedHDF5Writer == null ) driverVolumeWriterLocal.close(); } ); - /* - if ( this.downsamplings != null ) + + System.out.println( new Date( System.currentTimeMillis() ) + ": Saved full resolution, took: " + (System.currentTimeMillis() - time ) + " ms." ); + + // + // save multiresolution pyramid (s1 ... sN) + // + for ( int level = 1; level < mrInfo.length; ++level ) { - // TODO: run common downsampling code (affine, non-rigid, downsampling-only) - Downsampling.createDownsampling( - n5PathURI, - n5Dataset, - driverVolumeWriter, - dimensions, - storageType, - blockSize, - dataType, - compression, - downsamplings, - bdvString != null, - sc ); - } + final int s = level; + final List allBlocks = N5ApiTools.assembleJobs( mrInfo[ level ] ); - // close main writer (is shared over Spark-threads if it's HDF5, thus just closing it here) - driverVolumeWriter.close(); + System.out.println( new Date( System.currentTimeMillis() ) + ": Downsampling: " + Util.printCoordinates( mrInfo[ level ].absoluteDownsampling ) + " with relative downsampling of " + Util.printCoordinates( mrInfo[ level ].relativeDownsampling )); + System.out.println( new Date( System.currentTimeMillis() ) + ": s" + level + " num blocks=" + allBlocks.size() ); + System.out.println( new Date( System.currentTimeMillis() ) + ": Loading '" + mrInfo[ level - 1 ].dataset + "', downsampled will be written as '" + mrInfo[ level ].dataset + "'." ); - if ( multiRes ) - System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset.substring( 0, n5Dataset.length() - 3) + "'" ); - else - System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset + "'" ); + time = System.currentTimeMillis(); + + final JavaRDD rddDS = sc.parallelize( allBlocks ); - System.out.println( "done, took: " + (System.currentTimeMillis() - time ) + " ms." ); - */ + rddDS.foreach( + gridBlock -> + { + final N5Writer driverVolumeWriterLocal = N5Util.createN5Writer( outPathURI, storageType ); + + // 5D OME-ZARR CONTAINER + if ( storageType == StorageFormat.ZARR ) + { + N5ApiTools.writeDownsampledBlock5dOMEZARR( + driverVolumeWriterLocal, + mrInfo[ s ], + mrInfo[ s - 1 ], + gridBlock, + cIndex, + tIndex ); + } + else + { + N5ApiTools.writeDownsampledBlock( + driverVolumeWriterLocal, + mrInfo[ s ], + mrInfo[ s - 1 ], + gridBlock ); + } + + if ( N5Util.sharedHDF5Writer == null ) + driverVolumeWriterLocal.close(); + + }); + + System.out.println( new Date( System.currentTimeMillis() ) + ": Saved level s " + level + ", took: " + (System.currentTimeMillis() - time ) + " ms." ); + } } + // close main writer (is shared over Spark-threads if it's HDF5, thus just closing it here) + driverVolumeWriter.close(); + + /* + if ( multiRes ) + System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset.substring( 0, n5Dataset.length() - 3) + "'" ); + else + System.out.println( "Saved, e.g. view with './n5-view -i " + n5PathURI + " -d " + n5Dataset + "'" ); + */ + + System.out.println( "done, took: " + (System.currentTimeMillis() - totalTime ) + " ms." ); + sc.close(); return null; From c269ff4af4501b54abfa4273e3b459bb34d63b34 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 13:20:52 -0500 Subject: [PATCH 20/38] added prefetching --- .../bigstitcher/spark/SparkAffineFusion.java | 46 +++++++++++++++++-- .../spark/fusion/WriteSuperBlock.java | 33 +------------ .../spark/fusion/WriteSuperBlockMasks.java | 2 +- .../bigstitcher/spark/util/ViewUtil.java | 12 +++++ 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 756ad64..f25d997 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -31,6 +31,8 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -68,6 +70,8 @@ import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; +import net.preibisch.bigstitcher.spark.fusion.OverlappingBlocks; +import net.preibisch.bigstitcher.spark.fusion.OverlappingViews; import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlock; import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlockMasks; import net.preibisch.bigstitcher.spark.util.BDVSparkInstantiateViewSetup; @@ -378,10 +382,10 @@ else if ( dataType == DataType.UINT16 ) mrInfo = mrInfos[ c + t*numChannels ]; // using bigger blocksizes than being stored for efficiency (needed for very large datasets) - final int[] superBlockSize = new int[ 3 ]; - Arrays.setAll( superBlockSize, d -> blockSize[ d ] * blocksPerJob[ d ] ); + final int[] computeBlockSize = new int[ 3 ]; + Arrays.setAll( computeBlockSize, d -> blockSize[ d ] * blocksPerJob[ d ] ); final List grid = Grid.create(dimensions, - superBlockSize, + computeBlockSize, blockSize); System.out.println( "numJobs = " + grid.size() ); @@ -392,7 +396,6 @@ else if ( dataType == DataType.UINT16 ) long time = System.currentTimeMillis(); - //TODO: prefetchExecutor!! rdd.foreach( gridBlock -> { @@ -424,6 +427,41 @@ else if ( dataType == DataType.UINT16 ) type = new FloatType(); } + // + // PREFETCHING + // + + // The min coordinates of the block that this job renders (in pixels) + final int n = gridBlock[ 0 ].length; + final long[] superBlockOffset = new long[ n ]; + Arrays.setAll( superBlockOffset, d -> gridBlock[ 0 ][ d ] + bbMin[ d ] ); + + // The size of the block that this job renders (in pixels) + final long[] superBlockSize = gridBlock[ 1 ]; + + //TODO: prefetchExecutor!! + final long[] fusedBlockMin = new long[ n ]; + final long[] fusedBlockMax = new long[ n ]; + final Interval fusedBlock = FinalInterval.wrap( fusedBlockMin, fusedBlockMax ); + + // pre-filter views that overlap the superBlock + Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); + Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); + + final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, registrations, fusedBlock ); + final OverlappingBlocks overlappingBlocks = OverlappingBlocks.find( dataLocal, overlappingViews, fusedBlock ); + if ( overlappingBlocks.overlappingViews().isEmpty() ) + return; + + System.out.println( "Prefetching: " + overlappingBlocks.overlappingViews().size() + " blocks from the input data." ); + + final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); + overlappingBlocks.prefetch(prefetchExecutor); + prefetchExecutor.shutdown(); + + System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); + + // returns a zero-min interval final RandomAccessibleInterval img = BlkAffineFusion.init( conv, dataLocal.getSequenceDescription().getImgLoader(), diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java index fa76a62..b775a69 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java @@ -121,37 +121,6 @@ public WriteSuperBlock( this.firstTileWins = firstTileWins; } - /** - * Find all views among the given {@code viewIds} that overlap the given {@code interval}. - * The image interval of each view is transformed into world coordinates - * and checked for overlap with {@code interval}, with a conservative - * extension of 2 pixels in each direction. - * - * @param spimData contains bounds and registrations for all views - * @param viewIds which views to check - * @param interval interval in world coordinates - * @return views that overlap {@code interval} - */ - public static List findOverlappingViews( - final SpimData spimData, - final List viewIds, - final Interval interval ) - { - final List< ViewId > overlapping = new ArrayList<>(); - - // expand to be conservative ... - final Interval expandedInterval = Intervals.expand( interval, 2 ); - - for ( final ViewId viewId : viewIds ) - { - final Interval bounds = ViewUtil.getTransformedBoundingBox( spimData, viewId ); - if ( ViewUtil.overlaps( expandedInterval, bounds ) ) - overlapping.add( viewId ); - } - - return overlapping; - } - @Override public void call( final long[][] gridBlock ) throws Exception { @@ -212,7 +181,7 @@ public void call( final long[][] gridBlock ) throws Exception // pre-filter views that overlap the superBlock Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); - final List< ViewId > overlappingViews = findOverlappingViews( dataLocal, viewIds, fusedBlock ); + final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, fusedBlock ); final N5Writer executorVolumeWriter = N5Util.createN5Writer(n5PathURI, storageType);//URITools.instantiateN5Writer( storageType, n5PathURI );//N5Util.createWriter( n5Path, storageType ); final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); //Executors.newFixedThreadPool( N_PREFETCH_THREADS ); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java index 326ca6c..b7b7b9c 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java @@ -167,7 +167,7 @@ public void call( final long[][] gridBlock ) throws Exception Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); - final List< ViewId > overlappingViews = WriteSuperBlock.findOverlappingViews( dataLocal, viewIds, fusedBlock ); + final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, fusedBlock ); final N5Writer executorVolumeWriter = N5Util.createN5Writer(n5PathURI, storageType); //URITools.instantiateN5Writer( storageType, n5PathURI );//N5Util.createWriter( n5Path, storageType ); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/util/ViewUtil.java b/src/main/java/net/preibisch/bigstitcher/spark/util/ViewUtil.java index 0702950..5ec0231 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/util/ViewUtil.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/util/ViewUtil.java @@ -109,6 +109,18 @@ public static Interval getTransformedBoundingBox( final SpimData data, final Vie return Intervals.smallestContainingInterval( reg.getModel().estimateBounds( new FinalInterval( dim ) ) ); } + /** + * Get the estimated bounding box of the specified view in world coordinates. + * This transforms the image dimension for {@code viewId} with the {@code + * ViewRegistration} for {@code viewId}, and takes the bounding box. + */ + public static Interval getTransformedBoundingBox( final SpimData data, final ViewId viewId, final AffineTransform3D t ) throws IllegalArgumentException + { + final Dimensions dim = getDimensions( data, viewId ); + + return Intervals.smallestContainingInterval( t.estimateBounds( new FinalInterval( dim ) ) ); + } + public static String viewIdToString(final ViewId viewId) { return viewId == null ? null : "{\"setupId\": " + viewId.getViewSetupId() + ", \"timePointId\": " + viewId.getTimePointId() + "}"; From 8517747c6b8a04aa4731fa141e13fc60c73f260c Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 13:38:07 -0500 Subject: [PATCH 21/38] clean up old code --- .../bigstitcher/spark/SparkAffineFusion.java | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index f25d997..bee7214 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -21,7 +21,6 @@ */ package net.preibisch.bigstitcher.spark; -import java.io.IOException; import java.io.Serializable; import java.net.URI; import java.util.ArrayList; @@ -30,27 +29,22 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; -import org.janelia.scicomp.n5.zstandard.ZstandardCompression; -import ij.IJ; import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.sequence.SequenceDescription; import mpicbg.spim.data.sequence.ViewDescription; import mpicbg.spim.data.sequence.ViewId; -import net.imglib2.FinalDimensions; import net.imglib2.FinalInterval; import net.imglib2.Interval; import net.imglib2.RandomAccessible; @@ -69,29 +63,18 @@ import net.imglib2.util.Util; import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; -import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; import net.preibisch.bigstitcher.spark.fusion.OverlappingBlocks; import net.preibisch.bigstitcher.spark.fusion.OverlappingViews; -import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlock; -import net.preibisch.bigstitcher.spark.fusion.WriteSuperBlockMasks; -import net.preibisch.bigstitcher.spark.util.BDVSparkInstantiateViewSetup; -import net.preibisch.bigstitcher.spark.util.Downsampling; import net.preibisch.bigstitcher.spark.util.Import; import net.preibisch.bigstitcher.spark.util.N5Util; import net.preibisch.bigstitcher.spark.util.Spark; -import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.plugin.fusion.FusionGUI.FusionType; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; -import net.preibisch.mvrecon.process.export.ExportN5Api; import net.preibisch.mvrecon.process.fusion.blk.BlkAffineFusion; -import net.preibisch.mvrecon.process.fusion.transformed.FusedRandomAccessibleInterval.Fusion; import net.preibisch.mvrecon.process.fusion.transformed.TransformVirtual; -import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; -import net.preibisch.mvrecon.process.n5api.SpimData2Tools; -import net.preibisch.mvrecon.process.n5api.SpimData2Tools.InstantiateViewSetup; import picocli.CommandLine; import picocli.CommandLine.Option; import util.Grid; @@ -277,63 +260,6 @@ else if ( dataType == DataType.UINT16 ) } catch (Exception e ) {} - /* - final String n5Dataset = this.n5Dataset != null ? this.n5Dataset : N5ApiTools.createBDVPath( this.bdvString, 0, this.storageType ); - - // TODO: expose - final Compression compression = new ZstandardCompression( 3 );// new GzipCompression( 1 ); - - final double minIntensity = ( dataTypeFusion != DataTypeFusion.FLOAT32 ) ? this.minIntensity : 0; - - System.out.println( "Format being written: " + storageType ); - final N5Writer driverVolumeWriter = N5Util.createN5Writer( n5PathURI, storageType ); - - driverVolumeWriter.createDataset( - n5Dataset, - dimensions, - blockSize, - dataType, - compression ); - - // saving metadata if it is bdv-compatible (we do this first since it might fail) - if ( bdvString != null ) - { - // A Functional Interface that converts a ViewId to a ViewSetup, only called if the ViewSetup does not exist - final InstantiateViewSetup instantiate = - new BDVSparkInstantiateViewSetup( angleIds, illuminationIds, channelIds, tileIds ); - - final ViewId viewId = Import.getViewId( bdvString ); - - try - { - if ( SpimData2Tools.writeBDVMetaData( - driverVolumeWriter, - storageType, - dataType, - dimensions, - compression, - blockSize, - downsamplings, - viewId, - n5PathURI, - xmlOutURI, - instantiate ) == null ) - { - System.out.println( "Failed to write metadata for '" + n5Dataset + "'." ); - return null; - } - } - catch (SpimDataException | IOException e) - { - e.printStackTrace(); - System.out.println( "Failed to write metadata for '" + n5Dataset + "': " + e ); - return null; - } - - System.out.println( "Done writing BDV metadata."); - } - */ - final SparkConf conf = new SparkConf().setAppName("AffineFusion"); if (localSparkBindAddress) From 02ab79426b793c5447ba1643f77d38f2162041f0 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 13:49:37 -0500 Subject: [PATCH 22/38] start putting masks back in --- .../bigstitcher/spark/SparkAffineFusion.java | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index bee7214..bfb356c 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -98,11 +98,11 @@ public static enum DataTypeFusion @Option(names = "--blockScale", description = "how many blocks to use for a single processing step, e.g. 4,4,1 means for blockSize a 128,128,64 that each spark thread writes 512,512,64 (default: 2,2,1)") private String blockScaleString = "2,2,1"; - //@Option(names = { "--masks" }, description = "save only the masks (this will not fuse the images)") - //private boolean masks = false; + @Option(names = { "--masks" }, description = "save only the masks (this will not fuse the images)") + private boolean masks = false; - //@Option(names = "--maskOffset", description = "allows to make masks larger (+, the mask will include some background) or smaller (-, some fused content will be cut off), warning: in the non-isotropic coordinate space of the raw input images (default: 0.0,0.0,0.0)") - //private String maskOffset = "0.0,0.0,0.0"; + @Option(names = "--maskOffset", description = "allows to make masks larger (+, the mask will include some background) or smaller (-, some fused content will be cut off), warning: in the non-isotropic coordinate space of the raw input images (default: 0.0,0.0,0.0)") + private String maskOffset = "0.0,0.0,0.0"; @Option(names = { "--firstTileWins" }, description = "use firstTileWins fusion strategy (default: false - using weighted average blending fusion)") private boolean firstTileWins = false; @@ -353,10 +353,6 @@ else if ( dataType == DataType.UINT16 ) type = new FloatType(); } - // - // PREFETCHING - // - // The min coordinates of the block that this job renders (in pixels) final int n = gridBlock[ 0 ].length; final long[] superBlockOffset = new long[ n ]; @@ -374,32 +370,45 @@ else if ( dataType == DataType.UINT16 ) Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); - final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, registrations, fusedBlock ); - final OverlappingBlocks overlappingBlocks = OverlappingBlocks.find( dataLocal, overlappingViews, fusedBlock ); - if ( overlappingBlocks.overlappingViews().isEmpty() ) - return; - - System.out.println( "Prefetching: " + overlappingBlocks.overlappingViews().size() + " blocks from the input data." ); - - final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); - overlappingBlocks.prefetch(prefetchExecutor); - prefetchExecutor.shutdown(); - - System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); - - // returns a zero-min interval - final RandomAccessibleInterval img = BlkAffineFusion.init( - conv, - dataLocal.getSequenceDescription().getImgLoader(), - viewIds, - registrations, - dataLocal.getSequenceDescription().getViewDescriptions(), - firstTileWins ? FusionType.FIRST : FusionType.AVG_BLEND,//fusion.getFusionType(), - 1, // linear interpolation - null, // intensity correction - new BoundingBox( new FinalInterval( bbMin, bbMax ) ), - (RealType & NativeType)type, - blockSize ); + final RandomAccessibleInterval img; + + if ( masks ) + { + System.out.println( "Creating masks for block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); + img = null; + } + else + { + // + // PREFETCHING, TODO: should be part of BlkAffineFusion.init + // + final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, registrations, fusedBlock ); + final OverlappingBlocks overlappingBlocks = OverlappingBlocks.find( dataLocal, overlappingViews, fusedBlock ); + if ( overlappingBlocks.overlappingViews().isEmpty() ) + return; + + System.out.println( "Prefetching: " + overlappingBlocks.overlappingViews().size() + " blocks from the input data." ); + + final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); + overlappingBlocks.prefetch(prefetchExecutor); + prefetchExecutor.shutdown(); + + System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); + + // returns a zero-min interval + img = BlkAffineFusion.init( + conv, + dataLocal.getSequenceDescription().getImgLoader(), + viewIds, + registrations, + dataLocal.getSequenceDescription().getViewDescriptions(), + firstTileWins ? FusionType.FIRST : FusionType.AVG_BLEND,//fusion.getFusionType(), + 1, // linear interpolation + null, // intensity correction + new BoundingBox( new FinalInterval( bbMin, bbMax ) ), + (RealType & NativeType)type, + blockSize ); + } final long[] blockOffset, blockSizeExport, gridOffset; From b3dc08cc096fbcfe9f8d20fd6243717283db64c0 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 13:51:13 -0500 Subject: [PATCH 23/38] added missing class --- .../bigstitcher/spark/SparkAffineFusion.java | 15 ++++ .../spark/fusion/OverlappingViews.java | 78 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingViews.java diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index bfb356c..21085c4 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -376,6 +376,21 @@ else if ( dataType == DataType.UINT16 ) { System.out.println( "Creating masks for block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); img = null; + /* + rdd.foreach( new WriteSuperBlockMasks( + xmlURI, + preserveAnisotropy, + anisotropyFactor, + minBB, + n5PathURI, + n5Dataset, + storageType, + serializedViewIds, + uint8, + uint16, + maskOff, + blockSize ) ); + */ } else { diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingViews.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingViews.java new file mode 100644 index 0000000..90c3d0a --- /dev/null +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingViews.java @@ -0,0 +1,78 @@ +package net.preibisch.bigstitcher.spark.fusion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import mpicbg.spim.data.SpimData; +import mpicbg.spim.data.sequence.ViewId; +import net.imglib2.Interval; +import net.imglib2.realtransform.AffineTransform3D; +import net.imglib2.util.Intervals; +import net.preibisch.bigstitcher.spark.util.ViewUtil; + +public class OverlappingViews +{ + /** + * Find all views among the given {@code viewIds} that overlap the given {@code interval}. + * The image interval of each view is transformed into world coordinates + * and checked for overlap with {@code interval}, with a conservative + * extension of 2 pixels in each direction. + * + * @param spimData contains bounds and registrations for all views + * @param viewIds which views to check + * @param interval interval in world coordinates + * @return views that overlap {@code interval} + */ + public static List findOverlappingViews( + final SpimData spimData, + final List viewIds, + final Interval interval ) + { + final List< ViewId > overlapping = new ArrayList<>(); + + // expand to be conservative ... + final Interval expandedInterval = Intervals.expand( interval, 2 ); + + for ( final ViewId viewId : viewIds ) + { + final Interval bounds = ViewUtil.getTransformedBoundingBox( spimData, viewId ); + if ( ViewUtil.overlaps( expandedInterval, bounds ) ) + overlapping.add( viewId ); + } + + return overlapping; + } + + /** + * Find all views among the given {@code viewIds} that overlap the given {@code interval}. + * The image interval of each view is transformed into world coordinates + * and checked for overlap with {@code interval}, with a conservative + * extension of 2 pixels in each direction. + * + * @param spimData contains bounds and registrations for all views + * @param viewIds which views to check + * @param interval interval in world coordinates + * @return views that overlap {@code interval} + */ + public static List findOverlappingViews( + final SpimData spimData, + final List viewIds, + final HashMap< ViewId, AffineTransform3D > registrations, + final Interval interval ) + { + final List< ViewId > overlapping = new ArrayList<>(); + + // expand to be conservative ... + final Interval expandedInterval = Intervals.expand( interval, 2 ); + + for ( final ViewId viewId : viewIds ) + { + final Interval bounds = ViewUtil.getTransformedBoundingBox( spimData, viewId, registrations.get( viewId ) ); + if ( ViewUtil.overlaps( expandedInterval, bounds ) ) + overlapping.add( viewId ); + } + + return overlapping; + } +} From 509e39004b6807a55a58d0b9b046ab091fa0f029 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 14:42:40 -0500 Subject: [PATCH 24/38] first potentially functioning mask creation code --- .../bigstitcher/spark/SparkAffineFusion.java | 36 +-- .../fusion/GenerateComputeBlockMasks.java | 173 ++++++++++++ .../spark/fusion/WriteSuperBlock.java | 252 ------------------ .../spark/fusion/WriteSuperBlockMasks.java | 239 ----------------- 4 files changed, 194 insertions(+), 506 deletions(-) create mode 100644 src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java delete mode 100644 src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java delete mode 100644 src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 21085c4..60b0a36 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -63,6 +63,7 @@ import net.imglib2.util.Util; import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; +import net.preibisch.bigstitcher.spark.fusion.GenerateComputeBlockMasks; import net.preibisch.bigstitcher.spark.fusion.OverlappingBlocks; import net.preibisch.bigstitcher.spark.fusion.OverlappingViews; import net.preibisch.bigstitcher.spark.util.Import; @@ -199,6 +200,8 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt System.out.println( "Loaded " + mrInfos.length + " metadata object for fused " + storageType + " volume(s)" ); + final double[] maskOff = Import.csvStringToDoubleArray(maskOffset); + final SpimData2 dataGlobal = Spark.getJobSpimData2( xmlURI, 0 ); if ( dataGlobal == null ) @@ -336,21 +339,28 @@ else if ( dataType == DataType.UINT16 ) final Converter conv; final Type type; + final boolean uint8, uint16; if ( dataType == DataType.UINT8 ) { conv = new RealUnsignedByteConverter<>( minIntensity, maxIntensity ); type = new UnsignedByteType(); + uint8 = true; + uint16 = false; } else if ( dataType == DataType.UINT16 ) { conv = new RealUnsignedShortConverter<>( minIntensity, maxIntensity ); type = new UnsignedShortType(); + uint8 = false; + uint16 = true; } else { conv = null; type = new FloatType(); + uint8 = false; + uint16 = false; } // The min coordinates of the block that this job renders (in pixels) @@ -370,34 +380,30 @@ else if ( dataType == DataType.UINT16 ) Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); + final List< ViewId > overlappingViews = + OverlappingViews.findOverlappingViews( dataLocal, viewIds, registrations, fusedBlock ); + final RandomAccessibleInterval img; if ( masks ) { System.out.println( "Creating masks for block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); - img = null; - /* - rdd.foreach( new WriteSuperBlockMasks( - xmlURI, - preserveAnisotropy, - anisotropyFactor, - minBB, - n5PathURI, - n5Dataset, - storageType, - serializedViewIds, + + img = new GenerateComputeBlockMasks( + dataLocal, + registrations, + overlappingViews, + bbMin, + bbMax, uint8, uint16, - maskOff, - blockSize ) ); - */ + maskOff ).call( gridBlock ); } else { // // PREFETCHING, TODO: should be part of BlkAffineFusion.init // - final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, registrations, fusedBlock ); final OverlappingBlocks overlappingBlocks = OverlappingBlocks.find( dataLocal, overlappingViews, fusedBlock ); if ( overlappingBlocks.overlappingViews().isEmpty() ) return; diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java new file mode 100644 index 0000000..5f0660a --- /dev/null +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java @@ -0,0 +1,173 @@ +/*- + * #%L + * Spark-based parallel BigStitcher project. + * %% + * Copyright (C) 2021 - 2024 Developers. + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +package net.preibisch.bigstitcher.spark.fusion; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import mpicbg.spim.data.sequence.ViewId; +import net.imglib2.Cursor; +import net.imglib2.FinalInterval; +import net.imglib2.Interval; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.converter.Converters; +import net.imglib2.img.Img; +import net.imglib2.img.array.ArrayImgs; +import net.imglib2.realtransform.AffineTransform3D; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.view.Views; +import net.preibisch.bigstitcher.spark.util.ViewUtil; +import net.preibisch.mvrecon.fiji.spimdata.SpimData2; + +public class GenerateComputeBlockMasks +{ + private final long[] minBB, maxBB; + + private final SpimData2 data; + + private final HashMap< ViewId, AffineTransform3D > registrations; + + private final List< ViewId > overlappingViews; + + private final boolean uint8; + + private final boolean uint16; + + private final double[] maskOffset; + + public GenerateComputeBlockMasks( + final SpimData2 dataLocal, + final HashMap< ViewId, AffineTransform3D > registrations, + final List< ViewId > overlappingViews, + final long[] minBB, + final long[] maxBB, + final boolean uint8, + final boolean uint16, + final double[] maskOffset ) + { + this.data = dataLocal; + this.minBB = minBB; + this.maxBB = maxBB; + this.registrations = registrations; + this.overlappingViews = overlappingViews; + this.uint8 = uint8; + this.uint16 = uint16; + this.maskOffset = maskOffset; + } + + public RandomAccessibleInterval call( final long[][] gridBlock ) + { + final int n = minBB.length; + + // The min coordinates of the block that this job renders (in pixels) + final long[] superBlockOffset = new long[ n ]; + Arrays.setAll( superBlockOffset, d -> gridBlock[ 0 ][ d ] + minBB[ d ] ); + + // The size of the block that this job renders (in pixels) + final long[] superBlockSize = gridBlock[ 1 ]; + + //System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); + + // The min grid coordinate of the block that this job renders, in units of the output grid. + // Note, that the block that is rendered may cover multiple output grid cells. + //final long[] outputGridOffset = gridBlock[ 2 ]; + + // -------------------------------------------------------- + // initialization work that is happening in every job, + // independent of gridBlock parameters + // -------------------------------------------------------- + + final long[] fusedBlockMin = new long[ n ]; + final long[] fusedBlockMax = new long[ n ]; + final Interval fusedBlock = FinalInterval.wrap( fusedBlockMin, fusedBlockMax ); + + // pre-filter views that overlap the superBlock + Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); + Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); + + //final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( data, viewIds, fusedBlock ); + + final Img img = ArrayImgs.unsignedBytes( fusedBlock.dimensionsAsLongArray() ); + final RandomAccessibleInterval block = Views.translate( img, fusedBlockMin ); + + for ( final ViewId viewId : overlappingViews ) + { + final Cursor c = block.localizingCursor(); + final double[] l = new double[ 3 ]; + + final Interval dim = new FinalInterval( ViewUtil.getDimensions( data, viewId ) ); + final AffineTransform3D model = registrations.get( viewId ); + + final double[] min = new double[ 3 ]; + final double[] max = new double[ 3 ]; + + Arrays.setAll( min, d -> dim.min( d ) - maskOffset[ d ] ); + Arrays.setAll( max, d -> dim.max( d ) + maskOffset[ d ] ); + +A: while ( c.hasNext() ) + { + final UnsignedByteType t = c.next(); + + if ( t.get() > 0 ) + continue; + + c.localize(l); + model.applyInverse(l, l); + + for ( int d = 0; d < 3; ++d ) + if ( l[ d ] < min[ d ] || l[ d ] > max[ d ] ) + continue A; + + t.set( 255 ); + } + } + + final RandomAccessibleInterval fullImg = Views.interval( Views.expandZero( img ), minBB, maxBB ); + + if ( uint8 ) + { + return fullImg; + //N5Utils.saveBlock(img, executorVolumeWriter, n5Dataset, gridBlock[2]); + } + else if ( uint16 ) + { + return Converters.convertRAI( + fullImg, + (i, o) -> o.setInteger( i.get() > 0 ? 65535 : 0 ), + new UnsignedShortType()); + + //N5Utils.saveBlock(sourceUINT16, executorVolumeWriter, n5Dataset, gridBlock[2]); + } + else + { + return Converters.convertRAI( + img, + (i, o) -> o.set( i.get() > 0 ? 1.0f : 0.0f ), + new FloatType()); + + //N5Utils.saveBlock(sourceFloat, executorVolumeWriter, n5Dataset, gridBlock[2]); + } + } +} diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java deleted file mode 100644 index b775a69..0000000 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlock.java +++ /dev/null @@ -1,252 +0,0 @@ -/*- - * #%L - * Spark-based parallel BigStitcher project. - * %% - * Copyright (C) 2021 - 2024 Developers. - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ -package net.preibisch.bigstitcher.spark.fusion; - -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.spark.api.java.function.VoidFunction; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; - -import mpicbg.spim.data.SpimData; -import mpicbg.spim.data.registration.ViewRegistration; -import mpicbg.spim.data.registration.ViewRegistrations; -import mpicbg.spim.data.registration.ViewTransformAffine; -import mpicbg.spim.data.sequence.ViewId; -import net.imglib2.FinalInterval; -import net.imglib2.Interval; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.img.cell.CellGrid; -import net.imglib2.realtransform.AffineTransform3D; -import net.imglib2.type.NativeType; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedShortType; -import net.imglib2.type.numeric.real.FloatType; -import net.imglib2.util.Intervals; -import net.imglib2.util.Util; -import net.preibisch.bigstitcher.spark.blk.Fusion; -import net.preibisch.bigstitcher.spark.blk.FusionFirstWins; -import net.preibisch.bigstitcher.spark.blk.N5Helper; -import net.preibisch.bigstitcher.spark.util.N5Util; -import net.preibisch.bigstitcher.spark.util.Spark; -import net.preibisch.bigstitcher.spark.util.ViewUtil; -import net.preibisch.mvrecon.fiji.spimdata.SpimData2; - -public class WriteSuperBlock implements VoidFunction< long[][] > -{ - private static final long serialVersionUID = 4185504467908084954L; - - private final URI xmlURI; - - private final boolean preserveAnisotropy; - - private final double anisotropyFactor; - - private final long[] minBB; - - private final URI n5PathURI; - - private final String n5Dataset; - - private final StorageFormat storageType; - - private final int[][] serializedViewIds; - - private final boolean uint8; - - private final boolean uint16; - - private final double minIntensity; - - private final double range; - - private final int[] blockSize; - - private final boolean firstTileWins; - - public WriteSuperBlock( - final URI xmlURI, - final boolean preserveAnisotropy, - final double anisotropyFactor, - final long[] minBB, - final URI n5PathURI, - final String n5Dataset, - final String bdvString, - final StorageFormat storageType, - final int[][] serializedViewIds, - final boolean uint8, - final boolean uint16, - final double minIntensity, - final double range, - final int[] blockSize, - final boolean firstTileWins ) - { - this.xmlURI = xmlURI; - this.preserveAnisotropy = preserveAnisotropy; - this.anisotropyFactor = anisotropyFactor; - this.minBB = minBB; - this.n5PathURI = n5PathURI; - this.n5Dataset = n5Dataset; - this.storageType = storageType; - this.serializedViewIds = serializedViewIds; - this.uint8 = uint8; - this.uint16 = uint16; - this.minIntensity = minIntensity; - this.range = range; - this.blockSize = blockSize; - this.firstTileWins = firstTileWins; - } - - @Override - public void call( final long[][] gridBlock ) throws Exception - { - final int n = blockSize.length; - - // The min coordinates of the block that this job renders (in pixels) - final long[] superBlockOffset = new long[ n ]; - Arrays.setAll( superBlockOffset, d -> gridBlock[ 0 ][ d ] + minBB[ d ] ); - - // The size of the block that this job renders (in pixels) - final long[] superBlockSize = gridBlock[ 1 ]; - - System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); - - // The min grid coordinate of the block that this job renders, in units of the output grid. - // Note, that the block that is rendered may cover multiple output grid cells. - final long[] outputGridOffset = gridBlock[ 2 ]; - - - - - // -------------------------------------------------------- - // initialization work that is happening in every job, - // independent of gridBlock parameters - // -------------------------------------------------------- - - // custom serialization - final SpimData2 dataLocal = Spark.getSparkJobSpimData2(xmlURI); - final List< ViewId > viewIds = Spark.deserializeViewIds( serializedViewIds ); - - // If requested, preserve the anisotropy of the data (such that - // output data has the same anisotropy as input data) by prepending - // an affine to each ViewRegistration - if ( preserveAnisotropy ) - { - final AffineTransform3D aniso = new AffineTransform3D(); - aniso.set( - 1.0, 0.0, 0.0, 0.0, - 0.0, 1.0, 0.0, 0.0, - 0.0, 0.0, 1.0 / anisotropyFactor, 0.0 ); - final ViewTransformAffine preserveAnisotropy = new ViewTransformAffine( "preserve anisotropy", aniso ); - - final ViewRegistrations registrations = dataLocal.getViewRegistrations(); - for ( final ViewId viewId : viewIds ) - { - final ViewRegistration vr = registrations.getViewRegistration( viewId ); - vr.preconcatenateTransform( preserveAnisotropy ); - vr.updateModel(); - } - } - - - final long[] gridPos = new long[ n ]; - final long[] fusedBlockMin = new long[ n ]; - final long[] fusedBlockMax = new long[ n ]; - final Interval fusedBlock = FinalInterval.wrap( fusedBlockMin, fusedBlockMax ); - - // pre-filter views that overlap the superBlock - Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); - Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); - final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, fusedBlock ); - - final N5Writer executorVolumeWriter = N5Util.createN5Writer(n5PathURI, storageType);//URITools.instantiateN5Writer( storageType, n5PathURI );//N5Util.createWriter( n5Path, storageType ); - final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); //Executors.newFixedThreadPool( N_PREFETCH_THREADS ); - - final CellGrid blockGrid = new CellGrid( superBlockSize, blockSize ); - final int numCells = ( int ) Intervals.numElements( blockGrid.getGridDimensions() ); - for ( int gridIndex = 0; gridIndex < numCells; ++gridIndex ) - { - blockGrid.getCellGridPositionFlat( gridIndex, gridPos ); - blockGrid.getCellInterval( gridPos, fusedBlockMin, fusedBlockMax ); - - for ( int d = 0; d < n; ++d ) - { - gridPos[ d ] += outputGridOffset[ d ]; - fusedBlockMin[ d ] += superBlockOffset[ d ]; - fusedBlockMax[ d ] += superBlockOffset[ d ]; - } - // gridPos is now the grid coordinate in the N5 output - - // determine which Cells and Views we need to compute the fused block - final OverlappingBlocks overlappingBlocks = OverlappingBlocks.find( dataLocal, overlappingViews, fusedBlock ); - - if ( overlappingBlocks.overlappingViews().isEmpty() ) - continue; - - try ( AutoCloseable prefetched = overlappingBlocks.prefetch( prefetchExecutor ) ) - { - NativeType type; - if ( uint8 ) - type = new UnsignedByteType(); - else if ( uint16 ) - type = new UnsignedShortType(); - else - type = new FloatType(); - - final RandomAccessibleInterval< NativeType > source; - - if ( firstTileWins ) - { - source = FusionFirstWins.fuseVirtual( - dataLocal, - overlappingBlocks.overlappingViews(), - fusedBlock, - type, - minIntensity, - range ); - } - else - { - source = Fusion.fuseVirtual( - dataLocal, - overlappingBlocks.overlappingViews(), - fusedBlock, - type, - minIntensity, - range ); - } - - N5Helper.saveBlock( source, executorVolumeWriter, n5Dataset, gridPos ); - } - } - prefetchExecutor.shutdown(); - - // if it is not the shared HDF5 writer, then close - if ( N5Util.sharedHDF5Writer != executorVolumeWriter ) - executorVolumeWriter.close(); - } -} diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java deleted file mode 100644 index b7b7b9c..0000000 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/WriteSuperBlockMasks.java +++ /dev/null @@ -1,239 +0,0 @@ -/*- - * #%L - * Spark-based parallel BigStitcher project. - * %% - * Copyright (C) 2021 - 2024 Developers. - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ -package net.preibisch.bigstitcher.spark.fusion; - -import java.net.URI; -import java.util.Arrays; -import java.util.List; - -import org.apache.spark.api.java.function.VoidFunction; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.imglib2.N5Utils; -import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; - -import mpicbg.spim.data.registration.ViewRegistration; -import mpicbg.spim.data.registration.ViewRegistrations; -import mpicbg.spim.data.registration.ViewTransformAffine; -import mpicbg.spim.data.sequence.ViewId; -import net.imglib2.Cursor; -import net.imglib2.FinalInterval; -import net.imglib2.Interval; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.converter.Converters; -import net.imglib2.img.Img; -import net.imglib2.img.array.ArrayImgs; -import net.imglib2.realtransform.AffineTransform3D; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedShortType; -import net.imglib2.type.numeric.real.FloatType; -import net.imglib2.util.Util; -import net.imglib2.view.Views; -import net.preibisch.bigstitcher.spark.SparkAffineFusion; -import net.preibisch.bigstitcher.spark.util.N5Util; -import net.preibisch.bigstitcher.spark.util.Spark; -import net.preibisch.bigstitcher.spark.util.ViewUtil; -import net.preibisch.mvrecon.fiji.spimdata.SpimData2; -import util.URITools; - -public class WriteSuperBlockMasks implements VoidFunction< long[][] > -{ - - private static final long serialVersionUID = 1346166310488822467L; - - private final URI xmlURI; - - private final boolean preserveAnisotropy; - - private final double anisotropyFactor; - - private final long[] minBB; - - private final URI n5PathURI; - - private final String n5Dataset; - - private final StorageFormat storageType; - - private final int[][] serializedViewIds; - - private final boolean uint8; - - private final boolean uint16; - - private final double[] maskOffset; - - private final int[] blockSize; - - public WriteSuperBlockMasks( - final URI xmlURI, - final boolean preserveAnisotropy, - final double anisotropyFactor, - final long[] minBB, - final URI n5PathURI, - final String n5Dataset, - final StorageFormat storageType, - final int[][] serializedViewIds, - final boolean uint8, - final boolean uint16, - final double[] maskOffset, - final int[] blockSize ) - { - this.xmlURI = xmlURI; - this.preserveAnisotropy = preserveAnisotropy; - this.anisotropyFactor = anisotropyFactor; - this.minBB = minBB; - this.n5PathURI = n5PathURI; - this.n5Dataset = n5Dataset; - this.storageType = storageType; - this.serializedViewIds = serializedViewIds; - this.uint8 = uint8; - this.uint16 = uint16; - this.maskOffset = maskOffset; - this.blockSize = blockSize; - } - - @Override - public void call( final long[][] gridBlock ) throws Exception - { - final int n = blockSize.length; - - // The min coordinates of the block that this job renders (in pixels) - final long[] superBlockOffset = new long[ n ]; - Arrays.setAll( superBlockOffset, d -> gridBlock[ 0 ][ d ] + minBB[ d ] ); - - // The size of the block that this job renders (in pixels) - final long[] superBlockSize = gridBlock[ 1 ]; - - System.out.println( "Fusing block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); - - // The min grid coordinate of the block that this job renders, in units of the output grid. - // Note, that the block that is rendered may cover multiple output grid cells. - final long[] outputGridOffset = gridBlock[ 2 ]; - - // -------------------------------------------------------- - // initialization work that is happening in every job, - // independent of gridBlock parameters - // -------------------------------------------------------- - - // custom serialization - final SpimData2 dataLocal = Spark.getSparkJobSpimData2( xmlURI ); - final List< ViewId > viewIds = Spark.deserializeViewIds( serializedViewIds ); - - // If requested, preserve the anisotropy of the data (such that - // output data has the same anisotropy as input data) by prepending - // an affine to each ViewRegistration - if ( preserveAnisotropy ) - { - final AffineTransform3D aniso = new AffineTransform3D(); - aniso.set( - 1.0, 0.0, 0.0, 0.0, - 0.0, 1.0, 0.0, 0.0, - 0.0, 0.0, 1.0 / anisotropyFactor, 0.0 ); - final ViewTransformAffine preserveAnisotropy = new ViewTransformAffine( "preserve anisotropy", aniso ); - - final ViewRegistrations registrations = dataLocal.getViewRegistrations(); - for ( final ViewId viewId : viewIds ) - { - final ViewRegistration vr = registrations.getViewRegistration( viewId ); - vr.preconcatenateTransform( preserveAnisotropy ); - vr.updateModel(); - } - } - - final long[] fusedBlockMin = new long[ n ]; - final long[] fusedBlockMax = new long[ n ]; - final Interval fusedBlock = FinalInterval.wrap( fusedBlockMin, fusedBlockMax ); - - // pre-filter views that overlap the superBlock - Arrays.setAll( fusedBlockMin, d -> superBlockOffset[ d ] ); - Arrays.setAll( fusedBlockMax, d -> superBlockOffset[ d ] + superBlockSize[ d ] - 1 ); - - final List< ViewId > overlappingViews = OverlappingViews.findOverlappingViews( dataLocal, viewIds, fusedBlock ); - - final N5Writer executorVolumeWriter = N5Util.createN5Writer(n5PathURI, storageType); //URITools.instantiateN5Writer( storageType, n5PathURI );//N5Util.createWriter( n5Path, storageType ); - - final Img img = ArrayImgs.unsignedBytes( fusedBlock.dimensionsAsLongArray() ); - final RandomAccessibleInterval block = Views.translate( img, fusedBlockMin ); - - for ( final ViewId viewId : overlappingViews ) - { - final Cursor c = Views.iterable( block ).localizingCursor(); - final double[] l = new double[ 3 ]; - - final Interval dim = new FinalInterval( ViewUtil.getDimensions( dataLocal, viewId ) ); - final ViewRegistration vr = ViewUtil.getViewRegistration( dataLocal, viewId ); - final AffineTransform3D model = vr.getModel(); - - final double[] min = new double[ 3 ]; - final double[] max = new double[ 3 ]; - - Arrays.setAll( min, d -> dim.min( d ) - maskOffset[ d ] ); - Arrays.setAll( max, d -> dim.max( d ) + maskOffset[ d ] ); - -A: while ( c.hasNext() ) - { - final UnsignedByteType t = c.next(); - - if ( t.get() > 0 ) - continue; - - c.localize(l); - model.applyInverse(l, l); - - for ( int d = 0; d < 3; ++d ) - if ( l[ d ] < min[ d ] || l[ d ] > max[ d ] ) - continue A; - - t.set( 255 ); - } - } - - if ( uint8 ) - { - N5Utils.saveBlock(img, executorVolumeWriter, n5Dataset, gridBlock[2]); - } - else if ( uint16 ) - { - final RandomAccessibleInterval< UnsignedShortType > sourceUINT16 = - Converters.convertRAI( - img, - (i, o) -> o.setInteger( i.get() > 0 ? 65535 : 0 ), - new UnsignedShortType()); - - N5Utils.saveBlock(sourceUINT16, executorVolumeWriter, n5Dataset, gridBlock[2]); - } - else - { - final RandomAccessibleInterval< FloatType > sourceFloat = - Converters.convertRAI( - img, - (i, o) -> o.set( i.get() > 0 ? 1.0f : 0.0f ), - new FloatType()); - - N5Utils.saveBlock(sourceFloat, executorVolumeWriter, n5Dataset, gridBlock[2]); - } - - // if it is not the shared HDF5 writer, then close - if ( N5Util.sharedHDF5Writer != executorVolumeWriter ) - executorVolumeWriter.close(); - } -} From a5810278a82e3684871ebf7dbcdac12e5a02c3df Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 14:58:13 -0500 Subject: [PATCH 25/38] fix a few bugs in mask creation, works now --- .../bigstitcher/spark/SparkAffineFusion.java | 24 +++++++++++-------- .../fusion/GenerateComputeBlockMasks.java | 7 +++++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 60b0a36..de51358 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -301,7 +301,10 @@ else if ( dataType == DataType.UINT16 ) viewIds.add( new ViewId( viewId.getTimePointId(), viewId.getViewSetupId() ) ); }); - System.out.println( "Fusing " + viewIds.size() + " views for this 3D volume ... " ); + if ( masks ) + System.out.println( "Creating masks for " + viewIds.size() + " views of this 3D volume ... " ); + else + System.out.println( "Fusing " + viewIds.size() + " views for this 3D volume ... " ); final MultiResolutionLevelInfo[] mrInfo; @@ -389,15 +392,16 @@ else if ( dataType == DataType.UINT16 ) { System.out.println( "Creating masks for block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); - img = new GenerateComputeBlockMasks( - dataLocal, - registrations, - overlappingViews, - bbMin, - bbMax, - uint8, - uint16, - maskOff ).call( gridBlock ); + img = Views.zeroMin( + new GenerateComputeBlockMasks( + dataLocal, + registrations, + overlappingViews, + bbMin, + bbMax, + uint8, + uint16, + maskOff ).call( gridBlock ) ); } else { diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java index 5f0660a..efe4bf9 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/GenerateComputeBlockMasks.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; +import ij.ImageJ; import mpicbg.spim.data.sequence.ViewId; import net.imglib2.Cursor; import net.imglib2.FinalInterval; @@ -33,10 +34,13 @@ import net.imglib2.converter.Converters; import net.imglib2.img.Img; import net.imglib2.img.array.ArrayImgs; +import net.imglib2.img.display.imagej.ImageJFunctions; +import net.imglib2.multithreading.SimpleMultiThreading; import net.imglib2.realtransform.AffineTransform3D; import net.imglib2.type.numeric.integer.UnsignedByteType; import net.imglib2.type.numeric.integer.UnsignedShortType; import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Util; import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.util.ViewUtil; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; @@ -144,7 +148,8 @@ public RandomAccessibleInterval call( final long[][] gridBlock ) } } - final RandomAccessibleInterval fullImg = Views.interval( Views.expandZero( img ), minBB, maxBB ); + final RandomAccessibleInterval fullImg = + Views.interval( Views.extendZero( block ), minBB, maxBB ); if ( uint8 ) { From 293b9f790cf4edca9997b4a527b25660abf8f13f Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 16:14:31 -0500 Subject: [PATCH 26/38] add possibility to run partial fusions/mask creations --- .../bigstitcher/spark/SparkAffineFusion.java | 76 ++++++++++++++----- .../AbstractSelectableViews.java | 34 +++++++++ 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index de51358..3900b80 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -63,6 +63,7 @@ import net.imglib2.util.Util; import net.imglib2.view.Views; import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractInfrastructure; +import net.preibisch.bigstitcher.spark.abstractcmdline.AbstractSelectableViews; import net.preibisch.bigstitcher.spark.fusion.GenerateComputeBlockMasks; import net.preibisch.bigstitcher.spark.fusion.OverlappingBlocks; import net.preibisch.bigstitcher.spark.fusion.OverlappingViews; @@ -109,6 +110,33 @@ public static enum DataTypeFusion private boolean firstTileWins = false; + @Option(names = { "-t", "--timepointIndex" }, description = "specify a specific timepoint index of the output container that should be fused, usually you would also specify what --angleId, --tileId, ... or ViewIds -vi are being fused.") + private Integer timepointIndex = null; + + @Option(names = { "-c", "--channelIndex" }, description = "specify a specific channel index of the output container that should be fused, usually you would also specify what --angleId, --tileId, ... or ViewIds -vi are being fused.") + private Integer channelIndex = null; + + + // To specify what goes into the current 3D volume + @Option(names = { "--angleId" }, description = "list the angle ids that should be processed, you can find them in the XML, e.g. --angleId '0,1,2' (default: all angles)") + protected String angleIds = null; + + @Option(names = { "--tileId" }, description = "list the tile ids that should be processed, you can find them in the XML, e.g. --tileId '0,1,2' (default: all tiles)") + protected String tileIds = null; + + @Option(names = { "--illuminationId" }, description = "list the illumination ids that should be processed, you can find them in the XML, e.g. --illuminationId '0,1,2' (default: all illuminations)") + protected String illuminationIds = null; + + @Option(names = { "--channelId" }, description = "list the channel ids that should be processed, you can find them in the XML (usually just one when fusing), e.g. --channelId '0,1,2' (default: all channels)") + protected String channelIds = null; + + @Option(names = { "--timepointId" }, description = "list the timepoint ids that should be processed, you can find them in the XML (usually just one when fusing), e.g. --timepointId '0,1,2' (default: all time points)") + protected String timepointIds = null; + + @Option(names = { "-vi" }, description = "specifically list the view ids (time point, view setup) that should be fused into a single image, e.g. -vi '0,0' -vi '0,1' (default: all view ids)") + protected String[] vi = null; + + URI outPathURI = null; /** * Prefetching now works with a Executors.newCachedThreadPool(); @@ -124,6 +152,19 @@ public Void call() throws Exception System.exit( 0 ); } + if ( timepointIndex != null && channelIndex == null || timepointIndex == null && channelIndex != null ) + { + System.out.println( "You have to specify timepointId and channelId together, one alone does not work. timepointId =" + timepointIndex + ", channelId=" + channelIndex ); + return null; + } + + if ( timepointIndex == null && ( vi != null || timepointIds != null || channelIds != null || illuminationIds != null || tileIds != null || angleIds != null ) ) + + { + System.out.println( "You can only specify specify angles, tiles, ..., ViewIds if you provided a specific timepointIndex & channelIndex."); + return null; + } + this.outPathURI = URITools.toURI( outputPathURIString ); System.out.println( "Fused volume: " + outPathURI ); @@ -155,8 +196,19 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt final boolean bdv = fusionFormat.toLowerCase().contains( "BDV" ); final URI xmlURI = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/InputXML", URI.class ); - final int numTimepoints = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumTimepoints", int.class ); - final int numChannels = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumChannels", int.class ); + + final int numTimepoints, numChannels; + + if ( timepointIndex == null ) + { + numTimepoints = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumTimepoints", int.class ); + numChannels = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/NumChannels", int.class ); + } + else + { + System.out.println( "Overriding numChannels and numTimepoints from metadata, instead processing timepointIndex=" + timepointIndex + ", channelIndex=" + channelIndex + " only."); + numTimepoints = numChannels = 1; + } final long[] bbMin = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_min", long[].class ); final long[] bbMax = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/Boundingbox_max", long[].class ); @@ -217,15 +269,11 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt "The number of channels and timepoint in XML does not match the number in the export dataset.\n" + "You have to specify which ViewIds/Channels/Illuminations/Tiles/Angles/Timepoints should be fused into\n" + "a specific 3D volume in the fusion dataset:\n"); - // TODO: support that - /* - final ArrayList< ViewId > viewIdsGlobal = this.loadViewIds( dataGlobal ); + + viewIdsGlobal = AbstractSelectableViews.loadViewIds( dataGlobal, vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds ); if ( viewIdsGlobal == null || viewIdsGlobal.size() == 0 ) return null; - */ - - return null; } else { @@ -243,12 +291,6 @@ else if ( dataType == DataType.UINT16 ) else System.out.println( "Fusing to FLOAT32" ); - /* - final double[] maskOff = Import.csvStringToDoubleArray(maskOffset); - if ( masks ) - System.out.println( "Fusing ONLY MASKS! Mask offset: " + Util.printCoordinates( maskOff ) ); - */ - // // final variables for Spark // @@ -290,14 +332,14 @@ else if ( dataType == DataType.UINT16 ) System.out.println( "\nProcessing channel " + c + ", timepoint " + t ); System.out.println( "-----------------------------------" ); - final int tIndex = t; - final int cIndex = c; + final int tIndex = (timepointIndex == null) ? t : timepointIndex; + final int cIndex = (channelIndex == null) ? c : channelIndex; final ArrayList< ViewId > viewIds = new ArrayList<>(); viewIdsGlobal.forEach( viewId -> { final ViewDescription vd = sd.getViewDescription( viewId ); - if ( tpIdToTpIndex.get( vd.getTimePointId() ) == tIndex && chIdToChIndex.get( vd.getViewSetup().getChannel().getId() ) == cIndex ) + if ( timepointIndex != null || tpIdToTpIndex.get( vd.getTimePointId() ) == tIndex && chIdToChIndex.get( vd.getViewSetup().getChannel().getId() ) == cIndex ) viewIds.add( new ViewId( viewId.getTimePointId(), viewId.getViewSetupId() ) ); }); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractSelectableViews.java b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractSelectableViews.java index b389386..c3bb489 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractSelectableViews.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/abstractcmdline/AbstractSelectableViews.java @@ -54,6 +54,39 @@ public abstract class AbstractSelectableViews extends AbstractBasic implements C protected String[] vi = null; public ArrayList< ViewId > loadViewIds( final SpimData2 dataGlobal ) throws IllegalArgumentException + { + return loadViewIds(dataGlobal, vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds); + /*Import.validateInputParameters(vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds); + + // select views to process + ArrayList< ViewId > viewIdsGlobal = + Import.createViewIds( + dataGlobal, vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds); + + if ( viewIdsGlobal.size() == 0 ) + { + throw new IllegalArgumentException( "No views to be processed." ); + } + else + { + System.out.println( "The following ViewIds will be processed: "); + Collections.sort( viewIdsGlobal ); + for ( final ViewId v : viewIdsGlobal ) + System.out.print( "[" + v.getTimePointId() + "," + v.getViewSetupId() + "] " ); + System.out.println(); + } + + return viewIdsGlobal;*/ + } + + public static ArrayList< ViewId > loadViewIds( + final SpimData2 dataGlobal, + final String[] vi, + final String angleIds, + final String channelIds, + final String illuminationIds, + final String tileIds, + final String timepointIds ) throws IllegalArgumentException { Import.validateInputParameters(vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds); @@ -77,4 +110,5 @@ public ArrayList< ViewId > loadViewIds( final SpimData2 dataGlobal ) throws Ille return viewIdsGlobal; } + } From 8cef690cdca53f1afecd1ff4e538df2c9832aa12 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 16:36:52 -0500 Subject: [PATCH 27/38] fix text --- .../net/preibisch/bigstitcher/spark/SparkAffineFusion.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index 3900b80..fd37137 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -266,9 +266,9 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt dataGlobal.getSequenceDescription().getTimePoints().getTimePointsOrdered().size() != numTimepoints ) { System.out.println( - "The number of channels and timepoint in XML does not match the number in the export dataset.\n" - + "You have to specify which ViewIds/Channels/Illuminations/Tiles/Angles/Timepoints should be fused into\n" - + "a specific 3D volume in the fusion dataset:\n"); + "The number of channels and timepoint in XML does not match the number in the export dataset." + + "You have to specify which ViewIds/Channels/Illuminations/Tiles/Angles/Timepoints should be fused into" + + "a specific 3D volume in the fusion dataset:"); viewIdsGlobal = AbstractSelectableViews.loadViewIds( dataGlobal, vi, angleIds, channelIds, illuminationIds, tileIds, timepointIds ); From 61f68a3287946bb9b5ef23251c8c5585d4391798 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Fri, 31 Jan 2025 16:36:59 -0500 Subject: [PATCH 28/38] add create fusion container --- install | 1 + 1 file changed, 1 insertion(+) diff --git a/install b/install index 4fb220a..76be2af 100755 --- a/install +++ b/install @@ -124,6 +124,7 @@ install_command detect-interestpoints "net.preibisch.bigstitcher.spark.SparkInte install_command match-interestpoints "net.preibisch.bigstitcher.spark.SparkGeometricDescriptorMatching" install_command stitching "net.preibisch.bigstitcher.spark.SparkPairwiseStitching" install_command solver "net.preibisch.bigstitcher.spark.Solver" +install_command create-fusion-container "net.preibisch.bigstitcher.spark.CreateFusionContainer" install_command affine-fusion "net.preibisch.bigstitcher.spark.SparkAffineFusion" install_command nonrigid-fusion "net.preibisch.bigstitcher.spark.SparkNonRigidFusion" From 8b61518aa7b0481f1ff9cc850dc45d44f024fe83 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Sun, 2 Feb 2025 12:51:04 -0500 Subject: [PATCH 29/38] fix bug in fusion, converter was null --- .../bigstitcher/spark/SparkAffineFusion.java | 38 +++++++++++-------- .../spark/fusion/OverlappingBlocks.java | 2 + 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java index fd37137..66d4ecc 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/SparkAffineFusion.java @@ -36,11 +36,13 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import ij.ImageJ; import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.sequence.SequenceDescription; import mpicbg.spim.data.sequence.ViewDescription; @@ -52,6 +54,8 @@ import net.imglib2.converter.Converter; import net.imglib2.converter.RealUnsignedByteConverter; import net.imglib2.converter.RealUnsignedShortConverter; +import net.imglib2.img.display.imagej.ImageJFunctions; +import net.imglib2.multithreading.SimpleMultiThreading; import net.imglib2.realtransform.AffineTransform3D; import net.imglib2.type.NativeType; import net.imglib2.type.Type; @@ -75,6 +79,7 @@ import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox; import net.preibisch.mvrecon.process.fusion.blk.BlkAffineFusion; import net.preibisch.mvrecon.process.fusion.transformed.TransformVirtual; +import net.preibisch.mvrecon.process.interestpointregistration.pairwise.constellation.grouping.Group; import net.preibisch.mvrecon.process.n5api.N5ApiTools; import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo; import picocli.CommandLine; @@ -232,21 +237,23 @@ else if ( outputPathURIString.toLowerCase().endsWith( ".h5" ) || outPathURI.toSt System.out.println( "blockSize: " + Arrays.toString( blockSize ) ); System.out.println( "dataType: " + dataType ); - final double minIntensity, maxIntensity; - - if ( driverVolumeWriter.listAttributes( "Bigstitcher-Spark").containsKey( "MinIntensity" ) ) + double minI = Double.NaN, maxI = Double.NaN; + try { - minIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MinIntensity", double.class ); - maxIntensity = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MaxIntensity", double.class ); - - System.out.println( "minIntensity: " + minIntensity ); - System.out.println( "maxIntensity: " + maxIntensity ); + minI = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MinIntensity", double.class ); + maxI = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MaxIntensity", double.class ); } - else + catch ( N5Exception e ) { - minIntensity = maxIntensity = Double.NaN; + System.out.println( "Min/Max intensity not stored." ); } + final double minIntensity = minI; + final double maxIntensity = maxI; + + System.out.println( "minIntensity: " + minI ); + System.out.println( "maxIntensity: " + maxI ); + final MultiResolutionLevelInfo[][] mrInfos = driverVolumeWriter.getAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", MultiResolutionLevelInfo[][].class ); @@ -329,12 +336,12 @@ else if ( dataType == DataType.UINT16 ) for ( int c = 0; c < numChannels; ++c ) for ( int t = 0; t < numTimepoints; ++t ) { - System.out.println( "\nProcessing channel " + c + ", timepoint " + t ); - System.out.println( "-----------------------------------" ); - final int tIndex = (timepointIndex == null) ? t : timepointIndex; final int cIndex = (channelIndex == null) ? c : channelIndex; + System.out.println( "\nProcessing channel " + cIndex + ", timepoint " + tIndex ); + System.out.println( "-----------------------------------" ); + final ArrayList< ViewId > viewIds = new ArrayList<>(); viewIdsGlobal.forEach( viewId -> { final ViewDescription vd = sd.getViewDescription( viewId ); @@ -348,12 +355,13 @@ else if ( dataType == DataType.UINT16 ) else System.out.println( "Fusing " + viewIds.size() + " views for this 3D volume ... " ); + viewIds.forEach( vd -> System.out.println( Group.pvid( vd ) ) ); final MultiResolutionLevelInfo[] mrInfo; if ( storageType == StorageFormat.ZARR ) mrInfo = mrInfos[ 0 ]; else - mrInfo = mrInfos[ c + t*numChannels ]; + mrInfo = mrInfos[ cIndex + tIndex*numChannels ]; // using bigger blocksizes than being stored for efficiency (needed for very large datasets) final int[] computeBlockSize = new int[ 3 ]; @@ -454,7 +462,7 @@ else if ( dataType == DataType.UINT16 ) if ( overlappingBlocks.overlappingViews().isEmpty() ) return; - System.out.println( "Prefetching: " + overlappingBlocks.overlappingViews().size() + " blocks from the input data." ); + System.out.println( "Prefetching: " + overlappingBlocks.numPrefetchBlocks() + " block(s) from " + overlappingBlocks.overlappingViews().size() + " overlapping view(s) in the input data." ); final ExecutorService prefetchExecutor = Executors.newCachedThreadPool(); overlappingBlocks.prefetch(prefetchExecutor); diff --git a/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingBlocks.java b/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingBlocks.java index 8526300..5a5c355 100644 --- a/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingBlocks.java +++ b/src/main/java/net/preibisch/bigstitcher/spark/fusion/OverlappingBlocks.java @@ -120,6 +120,8 @@ private OverlappingBlocks( this.prefetchBlocks = prefetchBlocks; } + public int numPrefetchBlocks() { return prefetchBlocks.size(); } + /** * Result of {@link OverlappingBlocks#prefetch}. Holds strong * references to prefetched data, until it is {@link #close() From 25c542f69f7ad944107e8a84ed314e389be01b69 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 3 Feb 2025 10:09:23 -0500 Subject: [PATCH 30/38] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 67b6929..0422259 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ This package allows you to run compute-intense parts of BigStitcher distributed * `SparkGeometricDescriptorMatching`/`match-interestpoints` (perform pair-wise interest point matching) * `SparkPairwiseStitching`/`stitching` (run pairwise stitching between overlapping tiles) * `Solver`/`solver` (perform the global solve, works with interest points and stitching) -* `SparkAffineFusion`/`affine-fusion` (fuse the aligned dataset using affine models, including translation) +* `CreateFusionContainer`/`create-fusion-container` (set up the container into which the affine fusion will write it's data, now supports OME-ZARR) +* `SparkAffineFusion`/`affine-fusion` (fuse the aligned dataset using interpolated affine/rigid/translation models) * `SparkNonRigidFusion`/`nonrigid-fusion` (fuse the aligned dataset using non-rigid models) Additonally there are some utility methods: From 6bd6418c74b067595843a9d65ff7698bc4d67955 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 3 Feb 2025 10:11:40 -0500 Subject: [PATCH 31/38] Update README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0422259..a9c9bd0 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,21 @@ [![install4j](https://www.ej-technologies.com/images/product_banners/install4j_small.png)](https://www.ej-technologies.com/products/install4j/overview.html) -This package allows you to run compute-intense parts of BigStitcher distributed on your workstation, a cluster or the cloud using Apache Spark. The following modules are currently available in BigStitcher-Spark listed as `JavaClassName`/`cmd-line-tool-name` (you can find documentation below, but a good start is also to just check out the cmd-line arguments, they mostly follow the BigStitcher GUI; each module takes an existing XML): +This package allows you to run compute-intense parts of BigStitcher distributed on your workstation, a cluster or the cloud using Apache Spark. The following modules are currently available in BigStitcher-Spark listed as `JavaClassName`/**`cmd-line-tool-name`** (you can find documentation below, but a good start is also to just check out the cmd-line arguments, they mostly follow the BigStitcher GUI; each module takes an existing XML): -* `SparkResaveN5`/`resave` (resave an XML dataset you defined in BigStitcher - use virtual loading only - into N5 for processing) -* `SparkInterestPointDetection`/`detect-interestpoints` (detect interest points for alignment) -* `SparkGeometricDescriptorMatching`/`match-interestpoints` (perform pair-wise interest point matching) -* `SparkPairwiseStitching`/`stitching` (run pairwise stitching between overlapping tiles) -* `Solver`/`solver` (perform the global solve, works with interest points and stitching) -* `CreateFusionContainer`/`create-fusion-container` (set up the container into which the affine fusion will write it's data, now supports OME-ZARR) -* `SparkAffineFusion`/`affine-fusion` (fuse the aligned dataset using interpolated affine/rigid/translation models) -* `SparkNonRigidFusion`/`nonrigid-fusion` (fuse the aligned dataset using non-rigid models) +* `SparkResaveN5`/**`resave`** (resave an XML dataset you defined in BigStitcher - use virtual loading only - into N5 for processing) +* `SparkInterestPointDetection`/**`detect-interestpoints`** (detect interest points for alignment) +* `SparkGeometricDescriptorMatching`/**`match-interestpoints`** (perform pair-wise interest point matching) +* `SparkPairwiseStitching`/**`stitching`** (run pairwise stitching between overlapping tiles) +* `Solver`/**`solver`** (perform the global solve, works with interest points and stitching) +* `CreateFusionContainer`/**`create-fusion-container`** (set up the container into which the affine fusion will write it's data, now supports OME-ZARR) +* `SparkAffineFusion`/**`affine-fusion`** (fuse the aligned dataset using interpolated affine/rigid/translation models) +* `SparkNonRigidFusion`/**`nonrigid-fusion`** (fuse the aligned dataset using non-rigid models) Additonally there are some utility methods: -* `SparkDownsample`/`downsample` (perform downsampling of existing volumes) -* `ClearInterestPoints`/`clear-interestpoints` (clears interest points) -* `ClearRegistrations`/`clear-registrations` (clears registrations) +* `SparkDownsample`/**`downsample`** (perform downsampling of existing volumes) +* `ClearInterestPoints`/**`clear-interestpoints`** (clears interest points) +* `ClearRegistrations`/**`clear-registrations`** (clears registrations) ***Note: BigStitcher-Spark is designed to work hand-in-hand with BigStitcher.** You can always verify the results of each step BigStitcher-Spark step interactively using BigStitcher by simply opening the XML. You can of course also run certain steps in BigStitcher, and others in BigStitcher-Spark. Not all functionality is 100% identical between BigStitcher and BigStitcher-Spark; important differences in terms of capabilities is described in the respective module documentation below (typically BigStitcher-Spark supports a specific feature that was hard to implement in BigStitcher and vice-versa).* From 1573975bb50343b1ff5d9165a542ad3ff919cd95 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Mon, 3 Feb 2025 12:51:07 -0500 Subject: [PATCH 32/38] using multiview-reconstruction 5.0.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0e6da2f..9e3f369 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 10.6.3 2.3.5 - 5.0.4-SNAPSHOT + 5.0.4 2.3.1 From 65285e0153fc5e28975d2415c3b38e00f3eaa31b Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 13:30:51 -0500 Subject: [PATCH 33/38] Update README.md --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a9c9bd0..f7e04fd 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Additonally there are some utility methods: * [Match Interest Points](#ip-match) * [Solver](#solver) * [Affine Fusion](#affine-fusion) + * [Create Fusion Container](#create-fusion-container) + * [Run Affine Fusion](#run-affine-fusion) * [Non-Rigid Fusion](#nonrigid-fusion) Overview of the BigStitcher-Spark pipeline @@ -73,7 +75,7 @@ Please ask your sysadmin for help how to run it on your **cluster**, below are h `mvn clean package -P fatjar` builds `target/BigStitcher-Spark-0.0.1-SNAPSHOT.jar` for distribution. -For running the fatjar on the **cloud** check out services such as [Amazon EMR](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark.html). An implementations of image readers and writers that support cloud storage can be found [here](https://github.com/bigdataviewer/bigdataviewer-omezarr). Note that running it on the cloud is an ongoing effort with [@kgabor](https://github.com/kgabor), [@tpietzsch](https://github.com/tpietzsch) and the AWS team that currently works as a prototype but is further being optimized. We will provide an updated documentation in due time. Note that some modules support prefetching `--prefetch`, which is important for cloud execution due to its delays as it pre-loads all image blocks in parallel before processing. +BigStitcher-Spark is now fully "cloud-native". For running the fatjar on the **cloud** check out services such as [Amazon EMR](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark.html) and [Google Serverless Batches](https://cloud.google.com/dataproc-serverless/docs/quickstarts/spark-batch). Note that some modules support prefetching `--prefetch`, which is important for cloud execution due to its delays as it pre-loads all image blocks in parallel before processing. We will soon add detailled information on how to run the examples on both cloud platforms (it works - if you need help now, please contact @StephanPreibisch). ## Example Datasets @@ -217,6 +219,15 @@ When using interestpoints (for timeseries alignment with grouping all views of a ### Affine Fusion +## Create Fusion Container + +bla bla + +## Run Affine Fusion + +bla bla + + Performs **fusion using affine transformation models** computed by the [solve](#solver) (including translations) that are stored in the XML (*Warning: not tested on 2D*). By default the affine fusion will create an output image that contains all transformed input views/images. While this is good in some cases such as tiled stitching tasks, the output volume can be unnecessarily large for e.g. multi-view datasets. Thus, prior to running the fusion it might be useful to [**define a custom bounding box**](https://imagej.net/plugins/bigstitcher/boundingbox) in BigStitcher. A typical set of calls (because it is three channels) for affine fusion into a multi-resolution ZARR using only translations on the **stitching** dataset is (e.g. [this dataset](https://drive.google.com/file/d/1ajjk4piENbRrhPWlR6HqoUfD7U7d9zlZ/view?usp=sharing)): From dfdd3dd662b5699e9c58e2565d84f679710bf40b Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 13:33:13 -0500 Subject: [PATCH 34/38] Update README.md --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f7e04fd..bdb3a90 100644 --- a/README.md +++ b/README.md @@ -219,16 +219,15 @@ When using interestpoints (for timeseries alignment with grouping all views of a ### Affine Fusion -## Create Fusion Container +Performs **fusion using affine transformation models** computed by the [solve](#solver) (also supports translations, rigid, interpolated models) that are stored in the XML (*Warning: not tested on 2D*). By default the affine fusion will create an output image that encompasses all transformed input views/images. While this is good in some cases such as tiled stitching tasks, the output volume can be unnecessarily large for e.g. multi-view datasets. Thus, prior to running the fusion it might be useful to [**define a custom bounding box**](https://imagej.net/plugins/bigstitcher/boundingbox) in BigStitcher. -bla bla +#### Create Fusion Container -## Run Affine Fusion -bla bla +#### Run Affine Fusion +bla bla -Performs **fusion using affine transformation models** computed by the [solve](#solver) (including translations) that are stored in the XML (*Warning: not tested on 2D*). By default the affine fusion will create an output image that contains all transformed input views/images. While this is good in some cases such as tiled stitching tasks, the output volume can be unnecessarily large for e.g. multi-view datasets. Thus, prior to running the fusion it might be useful to [**define a custom bounding box**](https://imagej.net/plugins/bigstitcher/boundingbox) in BigStitcher. A typical set of calls (because it is three channels) for affine fusion into a multi-resolution ZARR using only translations on the **stitching** dataset is (e.g. [this dataset](https://drive.google.com/file/d/1ajjk4piENbRrhPWlR6HqoUfD7U7d9zlZ/view?usp=sharing)): From fbd8be41acf00f2642530250944263721415e7fb Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 13:45:27 -0500 Subject: [PATCH 35/38] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bdb3a90..0bf1b29 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,7 @@ Performs **fusion using affine transformation models** computed by the [solve](# #### Create Fusion Container +The first step in the fusion is to create an empty output container that also contains all the metadata and multi-resolution pyramids. By default an **OME-ZARR** is created, **N5** and **HDF5** are also supported, but HDF5 only if Spark is not run in a distributed fashion but multi-threaded on a local computer. #### Run Affine Fusion From abbbc6fa13596a55dca33ace183555ea89abb553 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 13:48:30 -0500 Subject: [PATCH 36/38] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bf1b29..57a8c1d 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,11 @@ Performs **fusion using affine transformation models** computed by the [solve](# #### Create Fusion Container -The first step in the fusion is to create an empty output container that also contains all the metadata and multi-resolution pyramids. By default an **OME-ZARR** is created, **N5** and **HDF5** are also supported, but HDF5 only if Spark is not run in a distributed fashion but multi-threaded on a local computer. +The first step in the fusion is to create an empty output container that also contains all the metadata and still empty multi-resolution pyramids. By default an **OME-ZARR** is created, **N5** and **HDF5** are also supported, but HDF5 only if Spark is not run in a distributed fashion but multi-threaded on a local computer. A typical call for creating an output container for e.g. the **stitching** dataset is (e.g. [this dataset](https://drive.google.com/file/d/1ajjk4piENbRrhPWlR6HqoUfD7U7d9zlZ/view?usp=sharing)): + +./create-fusion-container -x ~/SparkTest/Stitching/dataset.xml -o ~/SparkTest/Stitching/Stitching/fused-8bit.zarr --preserveAnisotropy --multiRes -d UINT8 + + #### Run Affine Fusion From de9d4414971360bef02463343ffcefa8e686934c Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 14:14:24 -0500 Subject: [PATCH 37/38] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57a8c1d..1624721 100644 --- a/README.md +++ b/README.md @@ -225,9 +225,13 @@ Performs **fusion using affine transformation models** computed by the [solve](# The first step in the fusion is to create an empty output container that also contains all the metadata and still empty multi-resolution pyramids. By default an **OME-ZARR** is created, **N5** and **HDF5** are also supported, but HDF5 only if Spark is not run in a distributed fashion but multi-threaded on a local computer. A typical call for creating an output container for e.g. the **stitching** dataset is (e.g. [this dataset](https://drive.google.com/file/d/1ajjk4piENbRrhPWlR6HqoUfD7U7d9zlZ/view?usp=sharing)): -./create-fusion-container -x ~/SparkTest/Stitching/dataset.xml -o ~/SparkTest/Stitching/Stitching/fused-8bit.zarr --preserveAnisotropy --multiRes -d UINT8 +./create-fusion-container -x ~/SparkTest/Stitching/dataset.xml -o ~/SparkTest/Stitching/Stitching/fused.zarr --preserveAnisotropy --multiRes -d UINT8 +This will create an output container that contains a 3D volume for all channels and timepoints present in the dataset. In the case of OME-ZARR, it is a single 5D container, for N5 and HDF5 it is a series of 3D datasets. *Note: if you do not want to export the entire project, or want to specify which views/images go into which volume, please check the details below.* +The fusion container for the [dataset that was aligned using interest points](https://drive.google.com/file/d/13b0UzWuvpT_qL7JFFuGY9WWm-VEiVNj7/view?usp=sharing) can be created in the same way, except that we choose to use the bounding box `embryo` that was specified using BigStitcher and we choose to save as an BDV/BigStitcher project using N5 as underlying export data format: + +./create-fusion-container -x ~/SparkTest/IP/dataset.xml -o ~/SparkTest/IP/fused.n5 -xo ~/SparkTest/IP/dataset-fused.xml -s N5 -b embryo --bdv --multiRes -d UINT8 #### Run Affine Fusion From 9c13775234a5432c4fe2c7df68bc0a851deb7142 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 4 Feb 2025 14:25:05 -0500 Subject: [PATCH 38/38] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1624721..0681b0d 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ The first step in the fusion is to create an empty output container that also co ./create-fusion-container -x ~/SparkTest/Stitching/dataset.xml -o ~/SparkTest/Stitching/Stitching/fused.zarr --preserveAnisotropy --multiRes -d UINT8 -This will create an output container that contains a 3D volume for all channels and timepoints present in the dataset. In the case of OME-ZARR, it is a single 5D container, for N5 and HDF5 it is a series of 3D datasets. *Note: if you do not want to export the entire project, or want to specify which views/images go into which volume, please check the details below.* +By default, this will create an output container that contains a 3D volume for all channels and timepoints present in the dataset. In the case of OME-ZARR, it is a single 5D container, for N5 and HDF5 it is a series of 3D datasets. ***Note: if you do NOT want to export the entire project, or want to specify fusion assignments (which views/images are fused into which volume), please check the details below. In short, you can specify the dimensions of the output container here, and the fusion assignments in the affine-fusion step below.*** The fusion container for the [dataset that was aligned using interest points](https://drive.google.com/file/d/13b0UzWuvpT_qL7JFFuGY9WWm-VEiVNj7/view?usp=sharing) can be created in the same way, except that we choose to use the bounding box `embryo` that was specified using BigStitcher and we choose to save as an BDV/BigStitcher project using N5 as underlying export data format: