From 60683101c109a9826e8765df876a51208a2c2e7a Mon Sep 17 00:00:00 2001 From: gaellafond Date: Wed, 14 Jul 2021 15:55:13 +0800 Subject: [PATCH] first commit --- .github/workflows/push.yml | 29 + .github/workflows/release.yml | 32 + .gitignore | 8 + LICENCE | 21 + README.md | 47 ++ maven-settings.xml | 19 + pom.xml | 74 +++ settings.xml | 19 + .../aims/netcdf/DownloadManagerGenerator.java | 53 ++ src/main/java/au/gov/aims/netcdf/Example.java | 259 ++++++++ .../java/au/gov/aims/netcdf/Generator.java | 416 +++++++++++++ .../gov/aims/netcdf/NcAnimateGenerator.java | 557 ++++++++++++++++++ .../netcdf/bean/AbstractNetCDFVariable.java | 63 ++ .../gov/aims/netcdf/bean/NetCDFDataset.java | 197 +++++++ .../netcdf/bean/NetCDFPointCoordinate.java | 121 ++++ .../netcdf/bean/NetCDFTimeDepthVariable.java | 22 + .../aims/netcdf/bean/NetCDFTimeVariable.java | 21 + .../gov/aims/netcdf/bean/NetCDFVariable.java | 19 + .../netcdf/bean/NetCDFVectorVariable.java | 29 + .../au/gov/aims/netcdf/GeneratorTest.java | 116 ++++ 20 files changed, 2122 insertions(+) create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 LICENCE create mode 100644 README.md create mode 100644 maven-settings.xml create mode 100644 pom.xml create mode 100644 settings.xml create mode 100644 src/main/java/au/gov/aims/netcdf/DownloadManagerGenerator.java create mode 100644 src/main/java/au/gov/aims/netcdf/Example.java create mode 100644 src/main/java/au/gov/aims/netcdf/Generator.java create mode 100644 src/main/java/au/gov/aims/netcdf/NcAnimateGenerator.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/AbstractNetCDFVariable.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFDataset.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFPointCoordinate.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeDepthVariable.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeVariable.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFVariable.java create mode 100644 src/main/java/au/gov/aims/netcdf/bean/NetCDFVectorVariable.java create mode 100644 src/test/java/au/gov/aims/netcdf/GeneratorTest.java diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..ce03a5a --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,29 @@ +# Workflow executed on "push" events. +name: push + +on: push + +jobs: + + # Run tests in Maven. + test: + runs-on: ubuntu-latest + + container: + image: maven:3-jdk-8-slim + + steps: + + # Retrieve the code from Github. + # Use branch "v2" of the "checkout" plugin + - uses: actions/checkout@v2 + + - name: Install dependencies + run: | + apt-get update + apt-get install -y libnetcdf-c++4 + + - name: Execute tests with Maven. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: mvn -B --settings settings.xml clean test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fd5a07f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +# Workflow executed on "release" events. +name: release + +on: + release: + types: [published] + +jobs: + + # Deploy to Github Packages. + deploy: + name: Deploy to Github Packages + + runs-on: ubuntu-latest + + container: + image: maven:3-jdk-8-slim + + steps: + + # Retrieve the code from Github. + - uses: actions/checkout@v2 + + - name: Install dependencies + run: | + apt-get update + apt-get install -y libnetcdf-c++4 + + - name: Publish to GitHub Packages. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: mvn -B -e --settings settings.xml deploy diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e72cd43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# IntelliJ files +.idea/ +*.iml +# OpenJDK bug: https://bugs.openjdk.java.net/browse/JDK-8214300 +.attach_pid* + +# Maven files +target/ diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..228ab8b --- /dev/null +++ b/LICENCE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Australian Institute of Marine Science + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..803ab03 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +This project can be used to generate very small NetCDF files, which can be included in projects to run unit tests. + +## Tools + +NetCDF files can be viewed using different tools. The most convenient ones are: +- [Panoply](https://www.giss.nasa.gov/tools/panoply/) +- [Dive](http://software.cmar.csiro.au/www/en/software/dive.html) + +**NOTE**: As of writing this documentation, CMAR have ceased to support the download link for **Dive**. +If you really want to use this tool, you can still find it on the +[Web Archive](https://web.archive.org/web/20170314023923/http://software.cmar.csiro.au/www/en/software/dive.html). + +## NetCDF variable naming conventions + +If you want to create your own NetCDF file, you may be interested in the following resources. + +- [NetCDF naming convention for `long-name` and `standard-name`](https://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch02s03.html) +- [Table of common NetCDF `standard-name`](https://cfconventions.org/Data/cf-standard-names/77/build/cf-standard-name-table.html) + +Some metadata information is hidden in unrelated attributes. Just have a look at the EDAL library source code +to see how `standard_name` and `long_name` can be used to infer metadata on the component variables of a vector variable. + +Package: uk.ac.rdg.resc.edal.dataset.cdm +Class: CdmDatasetFactory +Source code: +``` +private IdComponentEastNorth determineVectorIdAndComponent(String stdName) { + if (stdName.contains("eastward_")) { + return new IdComponentEastNorth(stdName.replaceFirst("eastward_", ""), true, true); + } else if (stdName.contains("northward_")) { + return new IdComponentEastNorth(stdName.replaceFirst("northward_", ""), false, true); + } else if (stdName.matches("u-.*component")) { + return new IdComponentEastNorth(stdName.replaceFirst("u-(.*)component", "$1"), true, + false); + } else if (stdName.matches("v-.*component")) { + return new IdComponentEastNorth(stdName.replaceFirst("v-(.*)component", "$1"), false, + false); + } else if (stdName.matches(".*x_.*velocity")) { + return new IdComponentEastNorth(stdName.replaceFirst("(.*)x_(.*velocity)", "$1$2"), + true, false); + } else if (stdName.matches(".*y_.*velocity")) { + return new IdComponentEastNorth(stdName.replaceFirst("(.*)y_(.*velocity)", "$1$2"), + false, false); + } + return null; +} +``` \ No newline at end of file diff --git a/maven-settings.xml b/maven-settings.xml new file mode 100644 index 0000000..0c7c82d --- /dev/null +++ b/maven-settings.xml @@ -0,0 +1,19 @@ + + + + + + github_aimsks + USERNAME + ${env.GITHUB_TOKEN} + + + + github_openaims + USERNAME + ${env.GITHUB_TOKEN} + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..7d45718 --- /dev/null +++ b/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + au.gov.aims + netcdf-generator + 0.2 + jar + This project was created to produce very small NetCDF files, + to be used in unit tests of projects like NcAnimate, + without increasing the project size considerably. + + + UTF-8 + + 1.8 + 1.8 + + 5.0.0-alpha3 + + + + + github_openaims + GitHub Open-AIMS repo + https://maven.pkg.github.com/open-AIMS/* + + + + + aims-ks.mvn-repo + AIMS Knowledge System MVN Repo + https://raw.githubusercontent.com/aims-ks/mvn-repo/master/ + + + + + + edu.ucar + cdm + ${netcdfVersion} + + + + edu.ucar + netcdf4 + ${netcdfVersion} + + + + log4j + log4j + 1.2.17 + + + + + junit + junit + 4.13.1 + test + + + + + + github + GitHub AIMS-KS Apache Maven Packages + https://maven.pkg.github.com/open-aims/netcdf-generator + + + diff --git a/settings.xml b/settings.xml new file mode 100644 index 0000000..0e71c41 --- /dev/null +++ b/settings.xml @@ -0,0 +1,19 @@ + + + + + + github + ${env.GITHUB_USERNAME} + ${env.GITHUB_TOKEN} + + + + github_aimsks + ${env.GITHUB_USERNAME} + ${env.GITHUB_TOKEN} + + + diff --git a/src/main/java/au/gov/aims/netcdf/DownloadManagerGenerator.java b/src/main/java/au/gov/aims/netcdf/DownloadManagerGenerator.java new file mode 100644 index 0000000..80d4b4c --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/DownloadManagerGenerator.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.File; + +/* + * Class used to generate small NetCDF files used with DownloadManager tests + */ + +public class DownloadManagerGenerator { + private static final DateTimeZone TIMEZONE_BRISBANE = DateTimeZone.forID("Australia/Brisbane"); + + public static void main(String ... args) throws Exception { + Generator netCDFGenerator = new Generator(); + + // For the DownloadManager + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2018, 10, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2018, 10, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2018-10.nc"), false); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2018, 11, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2018, 11, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2018-11.nc"), false); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2018, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2018, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2018-12.nc"), false); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2018, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2018, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2018-12_modified.nc"), false, 1000); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2019, 1, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2019, 1, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2019-01.nc"), false); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2019, 2, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2019, 2, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_simple_2019-02.nc"), false); + } +} diff --git a/src/main/java/au/gov/aims/netcdf/Example.java b/src/main/java/au/gov/aims/netcdf/Example.java new file mode 100644 index 0000000..06555ad --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/Example.java @@ -0,0 +1,259 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf; + +import ucar.ma2.Array; +import ucar.ma2.ArrayDouble; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Example of NetCDF file creation, inspired from an Unidata example: + * https://www.unidata.ucar.edu/software/netcdf-java/v4.6/tutorial/NetcdfWriting.html + * Old example: + * https://www.unidata.ucar.edu/software/netcdf-java/v4.6/tutorial/NetcdfFileWriteable.html + * + * Java DOC: + * https://www.unidata.ucar.edu/software/netcdf-java/v4.6/javadoc/ucar/nc2/NetcdfFileWriter.html + * + * NOTE: We are not using ucar.nc2.Structure since it only works with NetCDF 3. We want to generate NetCDF 4. + */ +public class Example implements Closeable { + private static final NetcdfFileWriter.Version NETCDF_VERSION = NetcdfFileWriter.Version.netcdf4; + + public NetcdfFileWriter writer; + + public static void main(String ... args) throws Exception { + File outputFile = new File("/tmp/example.nc"); + try (Example netCDFGenerator = new Example(outputFile)) { + //netCDFGenerator.generateGbrRainbow(); + netCDFGenerator.generateWind(); + } + } + + public Example(File outputFile) throws IOException { + this.writer = NetcdfFileWriter.createNew( + NETCDF_VERSION, + outputFile.getAbsolutePath() + ); + } + + public void close() throws IOException { + if (this.writer != null) { + IOException firstException = null; + + try { + this.writer.flush(); + } catch (IOException ex) { + firstException = ex; + } + + try { + this.writer.close(); + } catch (IOException ex) { + if (firstException == null) { + firstException = ex; + } + } + + this.writer = null; + + if (firstException != null) { + throw firstException; + } + } + } + + public void generateGbrRainbow() throws IOException, InvalidRangeException { + Dimension latDimension = this.writer.addDimension("lat", 3); + Dimension lonDimension = this.writer.addDimension("lon", 4); + Dimension timeDimension = this.writer.addUnlimitedDimension("time"); + + List latDimensions = new ArrayList(); + latDimensions.add(latDimension); + + Variable latVariable = this.writer.addVariable("lat", DataType.FLOAT, latDimensions); + this.writer.addVariableAttribute("lat", "units", "degrees_north"); + this.writer.addVariableAttribute("lat", "_CoordinateAxisType", "Lat"); + + List lonDimensions = new ArrayList(); + lonDimensions.add(lonDimension); + Variable lonVariable = this.writer.addVariable("lon", DataType.FLOAT, lonDimensions); + this.writer.addVariableAttribute("lon", "units", "degrees_east"); + this.writer.addVariableAttribute("lon", "_CoordinateAxisType", "Lon"); + + List timeDimensions = new ArrayList(); + timeDimensions.add(timeDimension); + this.writer.addVariable("time", DataType.INT, timeDimensions); + this.writer.addVariableAttribute("time", "units", "hours since 1990-01-01"); + this.writer.addVariableAttribute("time", "_CoordinateAxisType", "Time"); + + + String tempShortName = "temperature"; + DataType tempDataType = DataType.DOUBLE; + List tempDimensions = new ArrayList(); + tempDimensions.add(timeDimension); + tempDimensions.add(latDimension); + tempDimensions.add(lonDimension); + + Variable tempVariable = this.writer.addVariable(tempShortName, tempDataType, tempDimensions); + this.writer.addVariableAttribute(tempShortName, "units", "C"); + + this.writer.create(); + + this.writer.write(latVariable, Array.factory(DataType.FLOAT, new int [] {3}, new float[] {41, 40, 39})); + this.writer.write(lonVariable, Array.factory(DataType.FLOAT, new int [] {4}, new float[] {-109, -107, -105, -103})); + // Do not write time dimension. It will get added "frame" by "frame" + + ArrayDouble.D3 tempData = new ArrayDouble.D3(1, latDimension.getLength(), lonDimension.getLength()); + Array timeData = Array.factory(DataType.INT, new int[] {1}); + + // loop over each record + for (int time=0; time<10; time++) { + // make up some data for this record, using different ways to fill the data arrays. + timeData.setInt(timeData.getIndex(), time * 12); // 12 hours + + for (int lat=0; lat latDimensions = new ArrayList(); + latDimensions.add(latDimension); + Variable latVariable = this.writer.addVariable("lat", DataType.FLOAT, latDimensions); + this.writer.addVariableAttribute("lat", "units", "degrees_north"); + this.writer.addVariableAttribute("lat", "_CoordinateAxisType", "Lat"); + + List lonDimensions = new ArrayList(); + lonDimensions.add(lonDimension); + Variable lonVariable = this.writer.addVariable("lon", DataType.FLOAT, lonDimensions); + this.writer.addVariableAttribute("lon", "units", "degrees_east"); + this.writer.addVariableAttribute("lon", "_CoordinateAxisType", "Lon"); + + List timeDimensions = new ArrayList(); + timeDimensions.add(timeDimension); + this.writer.addVariable("time", DataType.INT, timeDimensions); + this.writer.addVariableAttribute("time", "units", "hours since 1990-01-01"); + this.writer.addVariableAttribute("time", "_CoordinateAxisType", "Time"); + + + String windUShortName = "wspeed_u"; + DataType windUDataType = DataType.DOUBLE; + List windUDimensions = new ArrayList(); + windUDimensions.add(timeDimension); + windUDimensions.add(latDimension); + windUDimensions.add(lonDimension); + + Variable windUVariable = this.writer.addVariable(windUShortName, windUDataType, windUDimensions); + this.writer.addVariableAttribute(windUShortName, "units", "ms-1"); + this.writer.addVariableAttribute(windUShortName, "standard_name", "eastward_wind"); + + + String windVShortName = "wspeed_v"; + DataType windVDataType = DataType.DOUBLE; + List windVDimensions = new ArrayList(); + windVDimensions.add(timeDimension); + windVDimensions.add(latDimension); + windVDimensions.add(lonDimension); + + Variable windVVariable = this.writer.addVariable(windVShortName, windVDataType, windVDimensions); + this.writer.addVariableAttribute(windVShortName, "units", "ms-1"); + this.writer.addVariableAttribute(windVShortName, "standard_name", "northward_wind"); + + + this.writer.create(); + + + this.writer.write(latVariable, Array.factory(DataType.FLOAT, new int [] {lats.length}, lats)); + this.writer.write(lonVariable, Array.factory(DataType.FLOAT, new int [] {lons.length}, lons)); + // Do not write time dimension here, it's a "ongoing" (unlimited) dimension. + // It gets added "frame" by "frame", as new data gets added to the file. + + ArrayDouble.D3 windUData = new ArrayDouble.D3(1, latDimension.getLength(), lonDimension.getLength()); + ArrayDouble.D3 windVData = new ArrayDouble.D3(1, latDimension.getLength(), lonDimension.getLength()); + + Array timeData = Array.factory(DataType.INT, new int[] {1}); + + // Loop over each record + for (int time=0; time<10; time++) { + // Make up some data for this record, using different ways to fill the data arrays. + timeData.setInt(timeData.getIndex(), time); + + for (int lat=0; lat + */ +package au.gov.aims.netcdf; + +import au.gov.aims.netcdf.bean.NetCDFDataset; +import au.gov.aims.netcdf.bean.AbstractNetCDFVariable; +import au.gov.aims.netcdf.bean.NetCDFTimeDepthVariable; +import au.gov.aims.netcdf.bean.NetCDFTimeVariable; +import au.gov.aims.netcdf.bean.NetCDFVariable; +import org.joda.time.DateTime; +import org.joda.time.Hours; +import ucar.ma2.Array; +import ucar.ma2.ArrayDouble; +import ucar.ma2.DataType; +import ucar.ma2.Index; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; + +/** + * Generate NetCDF file containing a single data hypercube. + * + * To simplify the library, some assumptions were made: + * - Every variable have the following dimensions: time, lat, lon + * - Values are type Double + * + * Unidata example: + * https://www.unidata.ucar.edu/software/netcdf-java/v4.6/tutorial/NetcdfWriting.html + * + * Java DOC: + * https://www.unidata.ucar.edu/software/netcdf-java/v4.6/javadoc/ucar/nc2/NetcdfFileWriter.html + * + * UCAR source code: + * https://github.com/Unidata/netcdf-java/tree/master/cdm/core/src/main/java/ucar + */ +public class Generator { + // Switch to "netcdf3" if you are getting error with the generation of NetCDF4 files + private static final NetcdfFileWriter.Version NETCDF_VERSION = NetcdfFileWriter.Version.netcdf4; + + // NOTE: Null value can be set using attribute "_FillValue", "missing_value", etc, + // by adding the following line (for example) in the definition of the variable: + // writer.addVariableAttribute(variableName, "_FillValue", 9999); + // https://www.unidata.ucar.edu/software/netcdf-java/v4.6/tutorial/NetcdfDataset.html + // or simply using Double.NaN (as in most NetCDF files). + // "_FillValue", "missing_value" have different meanings: + // Sometimes there is need for more than one value to represent different kinds of missing data. + // In this case, the user should use one or more other variable attributes for the different kinds + // of missing data. For example, it might be appropriate to use _FillValue to mean that data that + // was expected never appeared, but missing_value where the creator of the data intends data to be + // missing, as around an irregular region represented by a rectangular grid. + // https://www.bic.mni.mcgill.ca/users/sean/Docs/netcdf/guide.txn_59.html + private static final Double NULL_VALUE = Double.NaN; + + /** + * Generate a NetCDF file containing at least one data hypercube + * @param outputFile Where the NetCDF file will be saved. + * @param datasets Data to save in the file. Only specify one, + * unless you want multiple data hypercubes in the NetCDF file. + * @throws IOException + * @throws InvalidRangeException + */ + public void generate(File outputFile, NetCDFDataset ... datasets) throws IOException, InvalidRangeException { + // Validate arguments + if (outputFile == null) { + throw new IllegalArgumentException("No output file provided"); + } + if (datasets == null || datasets.length < 1) { + throw new IllegalArgumentException("No dataset provided"); + } + + // Instantiate the UCAR NetCDF writer (with a try-with-resource to ensure it gets closed) + try (NetcdfFileWriter writer = NetcdfFileWriter.createNew(NETCDF_VERSION, outputFile.getAbsolutePath())) { + + List bundleList = new ArrayList(); + + // Initialise the NetCDF header + // - Declare UCAR Dimensions + // - Declare UCAR Variables + int datasetCount = 0; + for (NetCDFDataset dataset : datasets) { + Bundle bundle = new Bundle(dataset); + bundleList.add(bundle); + + // Set attributes + for (Map.Entry attributeEntry : dataset.getGlobalAttributes().entrySet()) { + writer.addGlobalAttribute(attributeEntry.getKey(), attributeEntry.getValue()); + } + + // Create a unique name for the dimensions / variables (to prevent clashes between hypercubes) + bundle.latVariableName = "lat"; + bundle.lonVariableName = "lon"; + bundle.timeVariableName = "time"; + bundle.heightVariableName = "zc"; // as defined in eReefs NetCDF files + if (datasetCount > 0) { + bundle.latVariableName += datasetCount; + bundle.lonVariableName += datasetCount; + bundle.timeVariableName += datasetCount; + bundle.heightVariableName += datasetCount; + } + + float[] lats = bundle.lats; + float[] lons = bundle.lons; + double[] heights = bundle.heights; + + // Declare lat / lon / time dimensions + bundle.latDimension = writer.addDimension(bundle.latVariableName, lats.length); + bundle.lonDimension = writer.addDimension(bundle.lonVariableName, lons.length); + Dimension timeDimension = writer.addUnlimitedDimension(bundle.timeVariableName); + + bundle.heightDimension = null; + if (heights != null) { + bundle.heightDimension = writer.addDimension(bundle.heightVariableName, heights.length); + } + + // Coordinate axis attributes. + // It seems to work without them (except for the vertical axis). + // I added them for all axes to follow the documentation. + // https://www.unidata.ucar.edu/software/netcdf-java/v4.6/reference/CoordinateAttributes.html + // https://www.unidata.ucar.edu/software/netcdf-java/v4.6/tutorial/CoordinateAttributes.html + + // Declare dimension variables (seams redundant, but it's required) + List latDimensions = new ArrayList(); + latDimensions.add(bundle.latDimension); + bundle.latVariable = writer.addVariable(bundle.latVariableName, DataType.FLOAT, latDimensions); + writer.addVariableAttribute(bundle.latVariableName, "units", "degrees_north"); + writer.addVariableAttribute(bundle.latVariableName, "_CoordinateAxisType", "Lat"); + + List lonDimensions = new ArrayList(); + lonDimensions.add(bundle.lonDimension); + bundle.lonVariable = writer.addVariable(bundle.lonVariableName, DataType.FLOAT, lonDimensions); + writer.addVariableAttribute(bundle.lonVariableName, "units", "degrees_east"); + writer.addVariableAttribute(bundle.lonVariableName, "_CoordinateAxisType", "Lon"); + + List timeDimensions = new ArrayList(); + timeDimensions.add(timeDimension); + writer.addVariable(bundle.timeVariableName, DataType.INT, timeDimensions); + writer.addVariableAttribute(bundle.timeVariableName, "units", dataset.getTimeUnit()); + writer.addVariableAttribute(bundle.timeVariableName, "_CoordinateAxisType", "Time"); + + if (bundle.heightDimension != null) { + List heightDimensions = new ArrayList(); + heightDimensions.add(bundle.heightDimension); + bundle.heightVariable = writer.addVariable(bundle.heightVariableName, DataType.DOUBLE, heightDimensions); + writer.addVariableAttribute(bundle.heightVariableName, "units", "m"); + writer.addVariableAttribute(bundle.heightVariableName, "_CoordinateAxisType", "Height"); + writer.addVariableAttribute(bundle.heightVariableName, "_CoordinateZisPositive", "up"); + } + + // Declare data variables (such as temp, salt, current, etc) + // NOTE: This is the declaration only. The data will be added later. + for (AbstractNetCDFVariable variable : dataset) { + String variableName = variable.getName(); + DataType dataType = DataType.DOUBLE; + List varDimensions = new ArrayList(); + if ((variable instanceof NetCDFTimeVariable) || (variable instanceof NetCDFTimeDepthVariable)) { + varDimensions.add(timeDimension); + } + varDimensions.add(bundle.latDimension); + varDimensions.add(bundle.lonDimension); + if (bundle.heightDimension != null && (variable instanceof NetCDFTimeDepthVariable)) { + varDimensions.add(bundle.heightDimension); + } + + writer.addVariable(variableName, dataType, varDimensions); + + for (Map.Entry attributeEntry : variable.getAttributes().entrySet()) { + // Set variable attributes such as "units", "standard_name", etc + writer.addVariableAttribute(variableName, attributeEntry.getKey(), attributeEntry.getValue()); + } + } + + datasetCount++; + } + + + // Create the file and switch off "define mode": + // It's no longer possible to define dimensions / variables pass this point. + writer.create(); + + + for (Bundle bundle : bundleList) { + float[] lats = bundle.lats; + float[] lons = bundle.lons; + double[] heights = bundle.heights; + + // Write all the lat / lon / heights (depths) values that will be used with the data. + writer.write(bundle.latVariable, Array.factory(DataType.FLOAT, new int [] {lats.length}, lats)); + writer.write(bundle.lonVariable, Array.factory(DataType.FLOAT, new int [] {lons.length}, lons)); + if (heights != null) { + writer.write(bundle.heightVariable, Array.factory(DataType.DOUBLE, new int [] {heights.length}, heights)); + } + + // Create a list of all dates used across variables (some variables might have time gap) + Set allDateTime = new TreeSet(); + for (AbstractNetCDFVariable variable : bundle.dataset) { + allDateTime.addAll(variable.getDates()); + } + + // Write the time dimension data to the NetCDF file + if (!allDateTime.isEmpty()) { + // Initialise the data array for the time variable + Array timeData = Array.factory(DataType.INT, new int[] {1}); + + // Write all the dates to the NetCDF file + int recordIndex = 0; + Index timeIndex = timeData.getIndex(); + for (DateTime date : allDateTime) { + // Calculate the number of hours that elapsed since NetCDF epoch and the provided date + // (that's how dates are recorded in NetCDF files) + int timeOffset = Hours.hoursBetween(bundle.dataset.getTimeEpoch(), date).getHours(); + + // Set the time data for the current record + timeData.setInt(timeIndex, timeOffset); + + writer.write(bundle.timeVariableName, new int[] {recordIndex}, timeData); + recordIndex++; + } + } + + // Write each variable to the NetCDF file, one variable at the time. + for (AbstractNetCDFVariable abstractVariable : bundle.dataset) { + + if (abstractVariable instanceof NetCDFVariable) { + // Variables without time nor depth (such as bathymetry "botz") + ArrayDouble.D2 variableData = new ArrayDouble.D2( + bundle.latDimension.getLength(), bundle.lonDimension.getLength()); + + for (int latIndex=0; latIndex + */ +package au.gov.aims.netcdf; + +import au.gov.aims.netcdf.bean.NetCDFDataset; +import au.gov.aims.netcdf.bean.NetCDFTimeDepthVariable; +import au.gov.aims.netcdf.bean.NetCDFTimeVariable; +import au.gov.aims.netcdf.bean.NetCDFVariable; +import au.gov.aims.netcdf.bean.NetCDFVectorVariable; +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Hours; +import ucar.ma2.InvalidRangeException; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +/* + * Class used to generate small NetCDF files used with NcAnimate tests + * + * NetCDF file samples: + * https://www.unidata.ucar.edu/software/netcdf/examples/files.html + */ + +public class NcAnimateGenerator { + private static final Logger LOGGER = Logger.getLogger(NcAnimateGenerator.class); + private static final DateTimeZone TIMEZONE_BRISBANE = DateTimeZone.forID("Australia/Brisbane"); + + public static void main(String ... args) throws Exception { + Generator netCDFGenerator = new Generator(); + + + // GBR4 Hydro v2 + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2014, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2014, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_v2_2014-12-01.nc"), false); + + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2014, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2014, 12, 3, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_v2_2014-12-02_missingFrames.nc"), true); + + // To be used as a replacement for small.nc + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, + new DateTime(2010, 9, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2010, 9, 1, 2, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_v2_2010-09-01_00h00-02h00.nc"), true); + + + // GBR1 Hydro v2 + NcAnimateGenerator.generateGbr1v2(netCDFGenerator, + new DateTime(2014, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2014, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr1_2014-12-01.nc")); + + NcAnimateGenerator.generateGbr1v2(netCDFGenerator, + new DateTime(2014, 12, 2, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2014, 12, 3, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr1_2014-12-02.nc")); + + + // Multi-hypercubes of data + NcAnimateGenerator.generateGbr4v2MultiHypercubes(netCDFGenerator, + new DateTime(2000, 1, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2000, 1, 2, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_v2_2000-01-01_multiHypercubes.nc")); + + + // GBR4 BGC + NcAnimateGenerator.generateGbr4bgc(netCDFGenerator, + new DateTime(2014, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2015, 1, 1, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/gbr4_bgc_2014-12.nc")); + + + // NOAA + NcAnimateGenerator.generateNoaa(netCDFGenerator, + new DateTime(2014, 12, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2015, 1, 1, 0, 0, TIMEZONE_BRISBANE), + new File("/tmp/multi_1.glo_30m.dp.201412.nc"), + new File("/tmp/multi_1.glo_30m.hs.201412.nc")); + } + + public static void generateGbr4v2( + Generator netCDFGenerator, + DateTime startDate, + DateTime endDate, + File outputFile, + boolean missingData) throws IOException, InvalidRangeException { + NcAnimateGenerator.generateGbr4v2(netCDFGenerator, startDate, endDate, outputFile, missingData, 4280); + } + + public static void generateGbr4v2( + Generator netCDFGenerator, + DateTime startDate, + DateTime endDate, + File outputFile, + boolean missingData, + long seed) throws IOException, InvalidRangeException { + Random rng = new Random(seed); + + float[] lats = Generator.getCoordinates(-28, -7.6f, 15); // y + float[] lons = Generator.getCoordinates(142, 156, 10); // x + + // List of all depths found in GBR4 v2 files + double[] allDepths = {-3890, -3680, -3480, -3280, -3080, -2880, -2680, -2480, -2280, -2080, -1880, -1680, -1480, -1295, -1135, -990, -865, -755, -655, -570, -495, -430, -370, -315, -270, -235, -200, -170, -145, -120, -103, -88, -73, -60, -49, -39.5, -31, -23.75, -17.75, -12.75, -8.8, -5.55, -3, -1.5, -0.5, 0.5, 1.5}; + + // List of depths used in configs + double[] usedDepths = {-1.5, -17.75, -49, -103, -200, -315}; + + double[] depths = usedDepths; + + + NetCDFDataset dataset = new NetCDFDataset(); + dataset.setGlobalAttribute("metadata_link", "http://marlin.csiro.au/geonetwork/srv/eng/search?&uuid=72020224-f086-434a-bbe9-a222c8e5cf0d"); + dataset.setGlobalAttribute("title", "GBR4 Hydro"); + dataset.setGlobalAttribute("paramhead", "GBR 4km resolution grid"); + + NetCDFTimeDepthVariable tempVar = new NetCDFTimeDepthVariable("temp", "degrees C"); + tempVar.setAttribute("long_name", "Temperature"); + dataset.addVariable(tempVar); + + NetCDFTimeDepthVariable saltVar = new NetCDFTimeDepthVariable("salt", "PSU"); + saltVar.setAttribute("long_name", "Salinity"); + dataset.addVariable(saltVar); + + //NetCDFTimeDepthVariable RT_exposeVar = new NetCDFTimeDepthVariable("RT_expose", "DegC week"); + //dataset.addVariable(RT_exposeVar); + + NetCDFTimeVariable wspeed_uVar = new NetCDFTimeVariable("wspeed_u", "ms-1"); + wspeed_uVar.setAttribute("long_name", "eastward_wind"); + NetCDFTimeVariable wspeed_vVar = new NetCDFTimeVariable("wspeed_v", "ms-1"); + wspeed_vVar.setAttribute("long_name", "northward_wind"); + dataset.addVectorVariable(new NetCDFVectorVariable("wind", wspeed_uVar, wspeed_vVar)); + + NetCDFTimeDepthVariable uVar = new NetCDFTimeDepthVariable("u", "ms-1"); + uVar.setAttribute("long_name", "Eastward current"); + NetCDFTimeDepthVariable vVar = new NetCDFTimeDepthVariable("v", "ms-1"); + vVar.setAttribute("long_name", "Northward current"); + dataset.addVectorVariable(new NetCDFVectorVariable("sea_water_velocity", uVar, vVar)); + + //NetCDFTimeDepthVariable dhwVar = new NetCDFTimeDepthVariable("dhw", "DegC-week"); + //dataset.addVariable(dhwVar); + + //NetCDFTimeVariable etaVar = new NetCDFTimeVariable("eta", "metre"); + //dataset.addVariable(etaVar); + + //NetCDFTimeDepthVariable temp_exposeVar = new NetCDFTimeDepthVariable("temp_expose", "DegC week"); + //dataset.addVariable(temp_exposeVar); + + NetCDFVariable botzVar = new NetCDFVariable("botz", "metre"); + botzVar.setAttribute("long_name", "Depth of sea-bed"); + dataset.addVariable(botzVar); + + + int startHour = Hours.hoursBetween(dataset.getTimeEpoch(), startDate).getHours(); + int endHour = Hours.hoursBetween(dataset.getTimeEpoch(), endDate).getHours(); + for (float lat : lats) { + for (float lon : lons) { + // Set data for NetCDFVariable + double botzValue = lat % 10 + lon % 10; + botzVar.addDataPoint(lat, lon, botzValue); + + for (int hour=startHour; hour("wind", wspeed_uVar, wspeed_vVar)); + + NetCDFTimeDepthVariable uVar = new NetCDFTimeDepthVariable("u", "ms-1"); + uVar.setAttribute("long_name", "Eastward current"); + NetCDFTimeDepthVariable vVar = new NetCDFTimeDepthVariable("v", "ms-1"); + vVar.setAttribute("long_name", "Northward current"); + dataset.addVectorVariable(new NetCDFVectorVariable("sea_water_velocity", uVar, vVar)); + + NetCDFVariable botzVar = new NetCDFVariable("botz", "metre"); + botzVar.setAttribute("long_name", "Depth of sea-bed"); + dataset.addVariable(botzVar); + + + int startHour = Hours.hoursBetween(dataset.getTimeEpoch(), startDate).getHours(); + int endHour = Hours.hoursBetween(dataset.getTimeEpoch(), endDate).getHours(); + for (float lat : lats) { + for (float lon : lons) { + // Set data for NetCDFVariable + double botzValue = lat % 10 + lon % 10; + botzVar.addDataPoint(lat, lon, botzValue); + + for (int hour=startHour; hour("wind", wspeed_uVar, wspeed_vVar)); + + + NetCDFDataset dataset1 = new NetCDFDataset(); + + NetCDFTimeDepthVariable saltVar = new NetCDFTimeDepthVariable("salt", "PSU"); + saltVar.setAttribute("long_name", "Salinity"); + dataset1.addVariable(saltVar); + + NetCDFTimeDepthVariable uVar = new NetCDFTimeDepthVariable("u", "ms-1"); + uVar.setAttribute("long_name", "Eastward current"); + NetCDFTimeDepthVariable vVar = new NetCDFTimeDepthVariable("v", "ms-1"); + vVar.setAttribute("long_name", "Northward current"); + dataset1.addVectorVariable(new NetCDFVectorVariable("sea_water_velocity", uVar, vVar)); + + + int startHour0 = Hours.hoursBetween(dataset0.getTimeEpoch(), startDate).getHours(); + int endHour0 = Hours.hoursBetween(dataset0.getTimeEpoch(), endDate).getHours(); + for (float lat : lats0) { + for (float lon : lons0) { + for (int hour=startHour0; hour + */ +package au.gov.aims.netcdf.bean; + +import org.joda.time.DateTime; + +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +public class AbstractNetCDFVariable { + private String name; + + private Map attributes; + + private Map data; + + protected AbstractNetCDFVariable(String name, String units) { + this.name = name; + this.attributes = new HashMap(); + this.data = new HashMap(); + + this.setAttribute("units", units); + } + + public String getName() { + return this.name; + } + + public Map getAttributes() { + return this.attributes; + } + + public void setAttribute(String key, String value) { + this.attributes.put(key, value); + } + + + public Map getData() { + return this.data; + } + + public Double getValue(NetCDFPointCoordinate coordinate) { + return this.data.get(coordinate); + } + + public SortedSet getDates() { + SortedSet dates = new TreeSet(); + for (NetCDFPointCoordinate dataPoint : this.data.keySet()) { + if (dataPoint.getDate() != null) { + dates.add(dataPoint.getDate()); + } + } + return dates; + } + + public void addDataPoint(NetCDFPointCoordinate coordinate, Double value) { + this.data.put(coordinate, value); + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFDataset.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFDataset.java new file mode 100644 index 0000000..fd5c647 --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFDataset.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class NetCDFDataset implements Iterable { + private List variables; + private List vectorVariables; + private Map globalAttributes; + + // The date represented by time = 0, in NetCDF file + private String timeUnit; + private DateTime timeEpoch; + + public NetCDFDataset() { + this.variables = new ArrayList(); + this.vectorVariables = new ArrayList(); + this.globalAttributes = new HashMap(); + this.timeUnit = "hours since 1990-01-01"; + this.timeEpoch = new DateTime(1990, 1, 1, 0, 0, DateTimeZone.UTC); + } + + public String getTimeUnit() { + return this.timeUnit; + } + + public DateTime getTimeEpoch() { + return this.timeEpoch; + } + + public void setTimeUnit(String timeUnit, DateTime timeEpoch) { + this.timeUnit = timeUnit; + this.timeEpoch = timeEpoch; + } + + public Map getGlobalAttributes() { + return this.globalAttributes; + } + + public void setGlobalAttribute(String key, String value) { + this.globalAttributes.put(key, value); + } + + // Return a Dimensions instance containing all used lat, lon and heights. + // It needs to go through all data point coordinate, which takes times, + // but it's a small price to pay for the stability benefits. + public Dimensions getDimensions() { + Set latitudes = new HashSet(); + Set longitudes = new HashSet(); + Set heights = new HashSet(); + + for (AbstractNetCDFVariable variable : this) { + Map variableData = variable.getData(); + if (variableData != null && !variableData.isEmpty()) { + for (NetCDFPointCoordinate coordinate : variableData.keySet()) { + if (coordinate != null) { + latitudes.add(coordinate.getLat()); + longitudes.add(coordinate.getLon()); + + Double height = coordinate.getHeight(); + if (height != null) { + heights.add(height); + } + } + } + } + } + + return new Dimensions(latitudes, longitudes, heights); + } + + public List getVariables() { + return this.variables; + } + + public void setVariables(List variables) { + if (variables == null) { + this.variables.clear(); + } else { + this.variables = variables; + } + } + + public void addVariable(AbstractNetCDFVariable variable) { + this.variables.add(variable); + } + + + public List getVectorVariables() { + return this.vectorVariables; + } + + public void setVectorVariables(List vectorVariables) { + if (vectorVariables == null) { + this.vectorVariables.clear(); + } else { + this.vectorVariables = vectorVariables; + } + } + + public void addVectorVariable(NetCDFVectorVariable vectorVariable) { + this.vectorVariables.add(vectorVariable); + } + + + @Override + public Iterator iterator() { + List allVariables = new ArrayList(this.variables); + for (NetCDFVectorVariable vectorVariable : this.vectorVariables) { + allVariables.add(vectorVariable.getU()); + allVariables.add(vectorVariable.getV()); + } + + return allVariables.iterator(); + } + + public static class Dimensions { + private float[] latitudes; + private float[] longitudes; + private double[] heights; + + public Dimensions(Set latitudes, Set longitudes, Set heights) { + this(floatSetToArray(latitudes), floatSetToArray(longitudes), doubleSetToArray(heights)); + } + + public Dimensions(float[] latitudes, float[] longitudes, double[] heights) { + this.latitudes = latitudes; + this.longitudes = longitudes; + this.heights = heights; + + if (this.latitudes != null) { + Arrays.sort(this.latitudes); + } + if (this.longitudes != null) { + Arrays.sort(this.longitudes); + } + if (this.heights != null) { + Arrays.sort(this.heights); + } + } + + public float[] getLatitudes() { + return this.latitudes; + } + + public float[] getLongitudes() { + return this.longitudes; + } + + public double[] getHeights() { + return this.heights; + } + + + private static float[] floatSetToArray(Set floats) { + float[] floatArray = null; + if (floats != null && !floats.isEmpty()) { + floatArray = new float[floats.size()]; + int index = 0; + for (Float floatValue : floats) { + if (floatValue != null) { + floatArray[index] = floatValue; + index++; + } + } + } + return floatArray; + } + private static double[] doubleSetToArray(Set doubles) { + double[] doubleArray = null; + if (doubles != null && !doubles.isEmpty()) { + doubleArray = new double[doubles.size()]; + int index = 0; + for (Double doubleValue : doubles) { + if (doubleValue != null) { + doubleArray[index] = doubleValue; + index++; + } + } + } + return doubleArray; + } + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFPointCoordinate.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFPointCoordinate.java new file mode 100644 index 0000000..ac7ae91 --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFPointCoordinate.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +import org.joda.time.DateTime; + +import java.util.Objects; + +public class NetCDFPointCoordinate implements Comparable { + private static final float COORDINATE_EPSILON = 0.00001f; // about 1 metre on the equator + private static final double HEIGHT_EPSILON = 0.0000001; + + private float lat; + private float lon; + + private DateTime date; // Data date. Optional + private Double height; // Vertical coordinate axis. Optional + + public NetCDFPointCoordinate(float lat, float lon) { + this(lat, lon, null, null); + } + + public NetCDFPointCoordinate(float lat, float lon, DateTime date) { + this(lat, lon, date, null); + } + + public NetCDFPointCoordinate(float lat, float lon, DateTime date, Double height) { + this.lat = lat; + this.lon = lon; + this.date = date; + this.height = height; + } + + public float getLat() { + return this.lat; + } + + public float getLon() { + return this.lon; + } + + public DateTime getDate() { + return this.date; + } + + public Double getHeight() { + return this.height; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NetCDFPointCoordinate that = (NetCDFPointCoordinate) o; + return Float.compare(that.lat, this.lat) == 0 && + Float.compare(that.lon, this.lon) == 0 && + Objects.equals(this.date, that.date) && + Objects.equals(this.height, that.height); + } + + @Override + public int hashCode() { + return Objects.hash(this.lat, this.lon, this.date, this.height); + } + + @Override + public int compareTo(NetCDFPointCoordinate o) { + if (this == o) return 0; + + float latCmp = this.lat - o.lat; + if (latCmp > COORDINATE_EPSILON) { + return 1; + } + if (latCmp < -COORDINATE_EPSILON) { + return -1; + } + + float lonCmp = this.lon - o.lon; + if (lonCmp > COORDINATE_EPSILON) { + return 1; + } + if (lonCmp < -COORDINATE_EPSILON) { + return -1; + } + + if (this.date != o.date) { + if (this.date == null) { + return 1; + } + if (o.date == null) { + return -1; + } + + int dateCmp = this.date.compareTo(o.date); + if (dateCmp != 0) { + return dateCmp; + } + } + + if (this.height != o.height) { + if (this.height == null) { + return 1; + } + if (o.height == null) { + return -1; + } + + double heightCmp = this.height - o.height; + if (heightCmp > HEIGHT_EPSILON) { + return 1; + } + if (heightCmp < -HEIGHT_EPSILON) { + return -1; + } + } + + return 0; + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeDepthVariable.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeDepthVariable.java new file mode 100644 index 0000000..d233b28 --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeDepthVariable.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +import org.joda.time.DateTime; + +public class NetCDFTimeDepthVariable extends AbstractNetCDFVariable { + + public NetCDFTimeDepthVariable(String name, String units) { + super(name, units); + } + + public Double getValue(float lat, float lon, DateTime date, double height) { + return this.getValue(new NetCDFPointCoordinate(lat, lon, date, height)); + } + + public void addDataPoint(float lat, float lon, DateTime date, double height, double value) { + this.addDataPoint(new NetCDFPointCoordinate(lat, lon, date, height), value); + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeVariable.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeVariable.java new file mode 100644 index 0000000..e3fa26d --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFTimeVariable.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +import org.joda.time.DateTime; + +public class NetCDFTimeVariable extends AbstractNetCDFVariable { + public NetCDFTimeVariable(String name, String units) { + super(name, units); + } + + public Double getValue(float lat, float lon, DateTime date) { + return this.getValue(new NetCDFPointCoordinate(lat, lon, date)); + } + + public void addDataPoint(float lat, float lon, DateTime date, double value) { + this.addDataPoint(new NetCDFPointCoordinate(lat, lon, date), value); + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFVariable.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFVariable.java new file mode 100644 index 0000000..1c17e18 --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFVariable.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +public class NetCDFVariable extends AbstractNetCDFVariable { + public NetCDFVariable(String name, String units) { + super(name, units); + } + + public Double getValue(float lat, float lon) { + return this.getValue(new NetCDFPointCoordinate(lat, lon)); + } + + public void addDataPoint(float lat, float lon, double value) { + this.addDataPoint(new NetCDFPointCoordinate(lat, lon), value); + } +} diff --git a/src/main/java/au/gov/aims/netcdf/bean/NetCDFVectorVariable.java b/src/main/java/au/gov/aims/netcdf/bean/NetCDFVectorVariable.java new file mode 100644 index 0000000..e66547b --- /dev/null +++ b/src/main/java/au/gov/aims/netcdf/bean/NetCDFVectorVariable.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf.bean; + +public class NetCDFVectorVariable { + private String groupName; + private V u; + private V v; + + public NetCDFVectorVariable(String groupName, V u, V v) { + this.groupName = groupName; + + this.u = u; + this.u.setAttribute("standard_name", String.format("eastward_%s", this.groupName)); + + this.v = v; + this.v.setAttribute("standard_name", String.format("northward_%s", this.groupName)); + } + + public V getU() { + return this.u; + } + + public V getV() { + return this.v; + } +} diff --git a/src/test/java/au/gov/aims/netcdf/GeneratorTest.java b/src/test/java/au/gov/aims/netcdf/GeneratorTest.java new file mode 100644 index 0000000..4e47318 --- /dev/null +++ b/src/test/java/au/gov/aims/netcdf/GeneratorTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) Australian Institute of Marine Science, 2021. + * @author Gael Lafond + */ +package au.gov.aims.netcdf; + +import au.gov.aims.netcdf.bean.NetCDFDataset; +import au.gov.aims.netcdf.bean.NetCDFTimeVariable; +import au.gov.aims.netcdf.bean.NetCDFVariable; +import au.gov.aims.netcdf.bean.NetCDFVectorVariable; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.Hours; +import org.junit.Assert; +import org.junit.Test; +import ucar.ma2.InvalidRangeException; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +public class GeneratorTest { + private static final DateTimeZone TIMEZONE_BRISBANE = DateTimeZone.forID("Australia/Brisbane"); + + @Test + public void testGenerator() throws IOException, InvalidRangeException { + Generator netCDFGenerator = new Generator(); + File outputFile = new File("/tmp/test.nc"); + long expectedFileSize = 10 * 1024 * 1024; // 10 MB + + // Test file + GeneratorTest.generateTest(netCDFGenerator, + new DateTime(2019, 1, 1, 0, 0, TIMEZONE_BRISBANE), + new DateTime(2019, 1, 3, 0, 0, TIMEZONE_BRISBANE), + outputFile); + + Assert.assertTrue(String.format("The generated file doesn't exists or can not be read: %s", outputFile), + outputFile.canRead()); + + Assert.assertTrue(String.format("The generated file is smaller than expected.%n" + + "Expected: %9d%n" + + "Actual : %9d%n" + + "File : %s", + expectedFileSize, outputFile.length(), outputFile), + outputFile.length() > expectedFileSize); + + System.out.println(String.format("Load the generated file (%s) in a NetCDF viewer such as Panoply (%s) to check its integrity.", + outputFile, + "https://www.giss.nasa.gov/tools/panoply/")); + } + + /** + * Used to test this library + * @param netCDFGenerator The NetCDF file generator + * @param startDate The start date, inclusive + * @param endDate The end date, exclusive + * @param outputFile The location on disk where to save the NetCDF file + * @throws IOException + * @throws InvalidRangeException + */ + public static void generateTest( + Generator netCDFGenerator, + DateTime startDate, + DateTime endDate, + File outputFile) throws IOException, InvalidRangeException { + + Random rng = new Random(6930); + + float[] lats = Generator.getCoordinates(-50, 50, 100); + float[] lons = Generator.getCoordinates(-50, 50, 100); + + NetCDFDataset dataset = new NetCDFDataset(); + + NetCDFVariable botzVar = new NetCDFVariable("botz", "metre"); + dataset.addVariable(botzVar); + + NetCDFVariable botz2Var = new NetCDFVariable("botz2", "metre"); + dataset.addVariable(botz2Var); + + NetCDFTimeVariable testLinearGradient = new NetCDFTimeVariable("testLinearGradient", "Index"); + dataset.addVariable(testLinearGradient); + + NetCDFTimeVariable testRadialGradient = new NetCDFTimeVariable("testRadialGradient", "Index"); + dataset.addVariable(testRadialGradient); + + NetCDFTimeVariable testWaveU = new NetCDFTimeVariable("testWaveU", "m"); + NetCDFTimeVariable testWaveV = new NetCDFTimeVariable("testWaveV", "m"); + dataset.addVectorVariable(new NetCDFVectorVariable("testWave", testWaveU, testWaveV)); + + int nbHours = Hours.hoursBetween(startDate, endDate).getHours(); + for (float lat : lats) { + for (float lon : lons) { + double botzValue = lat % 10 + lon % 10; + botzVar.addDataPoint(lat, lon, botzValue); + botz2Var.addDataPoint(lat, lon, -botzValue); + + for (int hour=0; hour