From 474a50da0d99dcd0696a212d7704e81822384c6e Mon Sep 17 00:00:00 2001 From: Christopher J Lowrie Date: Thu, 4 Jan 2024 12:02:27 -0800 Subject: [PATCH 1/3] First commit, xb2zarr --- XB2Zarr/.gitignore | 1 + XB2Zarr/Dockerfile | 36 ++++ XB2Zarr/NetCDF_Multi_Function_Tool.py | 249 ++++++++++++++++++++++++++ XB2Zarr/README.md | 6 + XB2Zarr/app.py | 52 ++++++ XB2Zarr/gcsfuse_run.sh | 36 ++++ XB2Zarr/requirements.txt | 8 + XB2Zarr/tools/trigger.sh | 3 + 8 files changed, 391 insertions(+) create mode 100644 XB2Zarr/.gitignore create mode 100644 XB2Zarr/Dockerfile create mode 100644 XB2Zarr/NetCDF_Multi_Function_Tool.py create mode 100644 XB2Zarr/README.md create mode 100644 XB2Zarr/app.py create mode 100755 XB2Zarr/gcsfuse_run.sh create mode 100644 XB2Zarr/requirements.txt create mode 100644 XB2Zarr/tools/trigger.sh diff --git a/XB2Zarr/.gitignore b/XB2Zarr/.gitignore new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/XB2Zarr/.gitignore @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/XB2Zarr/Dockerfile b/XB2Zarr/Dockerfile new file mode 100644 index 0000000..5050f02 --- /dev/null +++ b/XB2Zarr/Dockerfile @@ -0,0 +1,36 @@ +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} + +# Install system dependencies +RUN set -e; \ + apt-get update -y && apt-get install -y \ + tini \ + lsb-release; \ + gcsFuseRepo=gcsfuse-`lsb_release -c -s`; \ + echo "deb https://packages.cloud.google.com/apt $gcsFuseRepo main" | \ + tee /etc/apt/sources.list.d/gcsfuse.list; \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + apt-key add -; \ + apt-get update; \ + apt-get install -y gcsfuse \ + && apt-get clean + +# Set fallback mount directory +ENV MNT_BASE=/mnt + +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +RUN pip3 install -r requirements.txt + +# Ensure the script is executable +RUN chmod +x /app/gcsfuse_run.sh + +# Use tini to manage zombie processes and signal forwarding +# https://github.com/krallin/tini +ENTRYPOINT ["/usr/bin/tini", "--"] + +# Pass the startup script as arguments to Tini +CMD ["/app/gcsfuse_run.sh"] \ No newline at end of file diff --git a/XB2Zarr/NetCDF_Multi_Function_Tool.py b/XB2Zarr/NetCDF_Multi_Function_Tool.py new file mode 100644 index 0000000..9f95461 --- /dev/null +++ b/XB2Zarr/NetCDF_Multi_Function_Tool.py @@ -0,0 +1,249 @@ +import os +import xarray as xr +import rioxarray +import numpy as np +import warnings +import rasterio +from PIL import Image +import numpy as np +import time +import dask +from dask.diagnostics import ProgressBar +# import matplotlib.pyplot as plt +warnings.filterwarnings("ignore", category=rasterio.errors.NotGeoreferencedWarning) + + + +start_time_total = time.time() + +start_time_idx = 0 +end_time_idx = 10 + +drive_letter = 'W:\\' +input_relative_path_reef_1s = r'Documents\Projects\Coastal_Resilience_Lab\04_USVI\01_Data\XBeach\Longreef\rp_100_1s\xboutput.nc' +input_relative_path_no_reef_1s = r'Documents\Projects\Coastal_Resilience_Lab\04_USVI\01_Data\XBeach\Longreef\No_Reef\rp_100_1s\xboutput.nc' +input_relative_path_reef_100s = r'Documents\Projects\Coastal_Resilience_Lab\04_USVI\01_Data\XBeach\Longreef\rp_100\xboutput.nc' +output_relative_path_general = r'Documents\Projects\Coastal_Resilience_Lab\04_USVI\03_Houdini\98_Input\XB' + + +def get_bounds(ds): + print(ds.isel(nx=0).globalx.min().compute()) + print(ds.isel(nx=-1).globalx.min().compute()) + + +def write_timestep_images(rds, full_output_path, start_time, end_time, variable, time_method): + + # Construct the full file path for the input and output + + # Define spacing for the grid + x_spacing = 10 # Adjust this value as needed + y_spacing = 10 # Adjust this value as needed + + # Open the dataset with Dask using xarray + # rds = xr.open_dataset(full_input_path, chunks={time_method: 500}) + + # Extract 'globalx', 'globaly', and the selected variable + globalx = rds['globalx'] + globaly = rds['globaly'] + selected_var = rds[variable] + + # Perform flattening and index calculations once + globalx_flat = globalx.values.flatten() + globaly_flat = globaly.values.flatten() + + x_min, x_max = np.min(globalx_flat), np.max(globalx_flat) + y_min, y_max = np.min(globaly_flat), np.max(globaly_flat) + num_points_x = int((x_max - x_min) / x_spacing) + 1 + num_points_y = int((y_max - y_min) / y_spacing) + 1 + + x_indices = ((globalx_flat - x_min) / x_spacing).astype(int) + y_indices = ((globaly_flat - y_min) / y_spacing).astype(int) + valid_indices = (x_indices >= 0) & (x_indices < num_points_x) & (y_indices >= 0) & (y_indices < num_points_y) + + # Create a template grid + template_grid = np.full((num_points_y, num_points_x), np.nan, dtype=float) + + # Iterate over each specified timestep + for timestep in range(start_time, end_time + 1): + print(f"Processing timestep {timestep}...") + + # Copy the template grid for current timestep + grid_values = template_grid.copy() + + # Extract data for the current timestep and flatten + timestep_flattened = selected_var.isel({time_method: timestep}).values.flatten() + + # Populate grid_values using pre-calculated indices + grid_values[y_indices[valid_indices], x_indices[valid_indices]] = timestep_flattened[valid_indices] + + # Replace NaN values with -1000 (or another value if needed) + grid_values[np.isnan(grid_values)] = -1000 + + # Save the image for the current timestep + tiff_filename = os.path.join(full_output_path, f'{variable}_timestep_{timestep}.tiff') + image = Image.fromarray(grid_values.astype('float32'), 'F') + image = image.transpose(Image.FLIP_TOP_BOTTOM) + image.save(tiff_filename, 'TIFF', compression=None) + + # # Save PNG Image + # png_filename = os.path.join(full_output_path, f'{variable}_timestep_{timestep}.png') + # image_png = Image.fromarray(normalized_values, 'L') + # image_png = image_png.transpose(Image.FLIP_TOP_BOTTOM) + # image_png.save(png_filename, 'PNG') + + print("Timesteps processed.") + + + +def write_variable_max(input_relative_path, output_relative_path, start_time, end_time): + + # Construct the full file paths + full_input_path = os.path.join(drive_letter, input_relative_path) + full_output_path = os.path.join(drive_letter, output_relative_path) + + # Extract the last folder name from the input relative path + last_folder_name = input_relative_path.split(os.sep)[-3] + + # Open the dataset without chunking to check dimensions + temp_rds = rioxarray.open_rasterio(full_input_path) + print("Dimensions of the dataset:", temp_rds.dims) + + # Open the dataset with Dask using xarray + print("Opening dataset with Dask using xarray...") + rds = xr.open_dataset(full_input_path, chunks={'globaltime': 500}) + print("Dataset opened with Dask.") + + # Extract 'globalx', 'globaly', and 'zs' + globalx = rds['globalx'] + globaly = rds['globaly'] + zs = rds['zs'] + + # Print initial dimensions + print("globalx dimensions:", globalx.shape) + print("globaly dimensions:", globaly.shape) + print("zs dimensions:", zs.shape) + + # Convert indices to float + start_time = float(zs['globaltime'].isel(globaltime=start_time).values) + end_time = float(zs['globaltime'].isel(globaltime=end_time).values) + + # Select the subset of data + subset_zs = zs.sel(globaltime=slice(start_time, end_time)) + + with ProgressBar(): + max_values = subset_zs.max(dim='globaltime', skipna=True).compute() + print("Maximum values calculated.") + + # Flatten the 'globalx' and 'globaly' arrays for the entire 2D space + globalx_flat = globalx.values.flatten() + globaly_flat = globaly.values.flatten() + print("globalx_flat dimensions:", globalx_flat.shape) + print("globaly_flat dimensions:", globaly_flat.shape) + print("Arrays flattened.") + + # Print max_values dimensions + print("max_values dimensions (before flatten):", max_values.shape) + max_values_flattened = max_values.values.flatten() + print("max_values dimensions (after flatten):", max_values_flattened.shape) + + # Calculate the extents (min and max values) of the flattened globalx and globaly + x_min = np.min(globalx_flat) + x_max = np.max(globalx_flat) + y_min = np.min(globaly_flat) + y_max = np.max(globaly_flat) + + # Specify the desired spacing between points for both axes + x_spacing = 10 # Adjust as needed + y_spacing = 10 # Adjust as needed + + # Calculate the number of points for each axis based on spacing + num_points_x = int((x_max - x_min) / x_spacing) + 1 + num_points_y = int((y_max - y_min) / y_spacing) + 1 + + # Create a 2D grid (grid_values) representing the values based on globalx and globaly + grid_values = np.full((num_points_y, num_points_x), np.nan, dtype=float) + + # Optimized code to populate grid_values + x_indices = ((globalx_flat - x_min) / x_spacing).astype(int) + y_indices = ((globaly_flat - y_min) / y_spacing).astype(int) + + valid_indices = (x_indices >= 0) & (x_indices < num_points_x) & (y_indices >= 0) & (y_indices < num_points_y) + + grid_values[y_indices[valid_indices], x_indices[valid_indices]] = max_values_flattened[valid_indices] + + # Replace NaN values + grid_values[np.isnan(grid_values)] = 0 + + # Save the image as a 16-bit TIFF + tiff_filename = os.path.join(full_output_path, f'Max_{last_folder_name}_{start_time_idx}_{end_time_idx}.tiff') + image = Image.fromarray(grid_values.astype('float32'), 'F') + image = image.transpose(Image.FLIP_TOP_BOTTOM) + image.save(tiff_filename, 'TIFF', compression=None) + print("Image saved.") + + # Return the array + return grid_values + + + +# Must be called directly after generating both max images, which populates the arrays to compare +def generate_difference_image(output_relative_path, start_time, end_time): + grayscale_output_path = os.path.join(drive_letter, output_relative_path, f'difference_image_{start_time}_{end_time}_grayscale.tiff') + color_output_path = os.path.join(drive_letter, output_relative_path, f'difference_image_{start_time}_{end_time}_color.tiff') + + # Ensure that both arrays have the same shape + # if reef_grid_values.shape != no_reef_grid_values.shape: + # raise ValueError("The two arrays must have the same dimensions for comparison.") + + # Calculate the difference between the two arrays (including both positive and negative differences) + difference_array = no_reef_grid_values - reef_grid_values + + # Flip the difference array vertically to correct the orientation + difference_array = np.flipud(difference_array) + + # Create a color difference image using a reversed colormap (coolwarm_r) with the specified range + cmap = plt.get_cmap('RdBu_r') + + # Normalize the colormap around zero + vmax = max(abs(difference_array.max()), abs(difference_array.min())) + vmin = -vmax + + # Create color difference image + rgba_image = cmap(np.clip((difference_array - vmin) / (vmax - vmin), 0, 1)) + + # Create grayscale image from the difference array + grayscale_difference_image = Image.fromarray(difference_array.astype('float32'), 'F') + + # Create color image from the RGBA array (as float32) + color_difference_image = Image.fromarray((rgba_image[:, :, :3] * 255).astype('uint8'), 'RGB') + + # Print the minimum and maximum values of the difference array + print("Minimum value in difference array:", difference_array.min()) + print("Maximum value in difference array:", difference_array.max()) + + # Save the grayscale and color difference images as TIFF + grayscale_difference_image.save(grayscale_output_path, 'TIFF', compression=None) + color_difference_image.save(color_output_path, 'TIFF', compression=None) + + + +# Call write timesteps zs +#write_timestep_images(input_relative_path=input_relative_path_reef_1s, output_relative_path=output_relative_path_general, start_time=start_time_idx, end_time=end_time_idx, variable='zs', time_method='globaltime') + +# Call write timesteps zb_mean +# write_timestep_images(input_relative_path=input_relative_path_reef_100s, output_relative_path=output_relative_path_general, start_time=0, end_time=0, variable='zb_mean', time_method='meantime') + +# # Call write_variable_max +# # Note, currently if called multiple times, it will overwrite the output. Only call multiple times if needed for the generate_difference_image function immediately after +# reef_grid_values = write_variable_max(input_relative_path=input_relative_path_reef_1s, output_relative_path=output_relative_path_general, start_time=start_time_idx, end_time=end_time_idx) + +# no_reef_grid_values = write_variable_max(input_relative_path=input_relative_path_no_reef_1s, output_relative_path=output_relative_path_general, start_time=start_time_idx, end_time=end_time_idx) + +# # Call generate_difference_image with grid values +# generate_difference_image(output_relative_path=output_relative_path_general, start_time=start_time_idx, end_time=end_time_idx) + + + +# # End the timer and print the total runtime +# end_time_total = time.time() +# print(f"Total runtime: {end_time_total - start_time_total} seconds") diff --git a/XB2Zarr/README.md b/XB2Zarr/README.md new file mode 100644 index 0000000..e601ffc --- /dev/null +++ b/XB2Zarr/README.md @@ -0,0 +1,6 @@ +### Local Testing +ENV=dev +BASE_GAR_DIRECTORY=us-west1-docker.pkg.dev/global-mangroves +BASE_IMAGE=${BASE_GAR_DIRECTORY}/base/python_gis_base_${ENV} +docker build -t xb2zarr --build-arg BASE_IMAGE=$BASE_IMAGE . +docker run -it -v $HOME/.config/gcloud:/root/.config/gcloud -e OUTPUT_BUCKET=geopmaker-output-staging -p 3002:8080 -v $PWD:/app xb2zarr \ No newline at end of file diff --git a/XB2Zarr/app.py b/XB2Zarr/app.py new file mode 100644 index 0000000..c8f06f8 --- /dev/null +++ b/XB2Zarr/app.py @@ -0,0 +1,52 @@ +from typing import Annotated + +from fastapi import FastAPI, File, Form, UploadFile +from fastapi.responses import FileResponse + +import os, uvicorn, io +import xarray as xr +import rioxarray as rxr +from NetCDF_Multi_Function_Tool import write_timestep_images, get_bounds +import uuid, shutil + +app = FastAPI() + + +@app.post("/files/") +async def create_file( + xboutput: Annotated[bytes, File()], + id: Annotated[str, Form()], +): + xid = str(uuid.uuid1()) + output_dir = os.path.join("/tmp", xid) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + ds = xr.open_dataset(io.BytesIO(xboutput), chunks={"globaltime": 500}) + print(get_bounds(ds)) + return + write_timestep_images(ds, output_dir, 0, 2, "zs", "globaltime") + print(os.listdir(output_dir)) + print("file written, sending...") + if not os.path.exists("/results"): + os.makedirs("/results") + results_dir = f"/results/{xid}" + if not os.path.exists(results_dir): + os.makedirs(results_dir) + output_zarr = os.path.join(output_dir, f"{id}.zarr") + datasets = [ + rxr.open_rasterio(os.path.join(output_dir, i))\ + .expand_dims({"time": 1})\ + .assign_coords({"time": [int(i.split('.')[0].split('_')[-1])]}) + for i in os.listdir(output_dir) + ] + print(datasets[0]) + xr.concat( + datasets, + dim="time" + ).to_zarr(output_zarr) + shutil.make_archive(os.path.join(results_dir, "results"), "zip", output_dir) + return FileResponse(os.path.join(results_dir, "results.zip")) + + +if __name__ == "__main__": + uvicorn.run("app:app", host="0.0.0.0", port=8080, reload=True) diff --git a/XB2Zarr/gcsfuse_run.sh b/XB2Zarr/gcsfuse_run.sh new file mode 100755 index 0000000..951a3e7 --- /dev/null +++ b/XB2Zarr/gcsfuse_run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [START cloudrun_fuse_script] +#!/usr/bin/env bash +set -eo pipefail + +echo "Mounting GCS Fuse." +for bucket in $(echo $MNT_BUCKETS | tr ";" "\n") +do + echo ["$bucket"] + mkdir -p $MNT_BASE/$bucket + # gcsfuse --implicit-dirs --debug_gcs --debug_fuse $bucket $MNT_BASE/$bucket + gcsfuse --implicit-dirs $bucket $MNT_BASE/$bucket +done + +echo "Mounting completed." + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. +exec python3 app.py +# [END cloudrun_fuse_script] \ No newline at end of file diff --git a/XB2Zarr/requirements.txt b/XB2Zarr/requirements.txt new file mode 100644 index 0000000..3cba516 --- /dev/null +++ b/XB2Zarr/requirements.txt @@ -0,0 +1,8 @@ +fastapi +python-multipart +uvicorn +scipy +h5netcdf +pillow +dask +zarr \ No newline at end of file diff --git a/XB2Zarr/tools/trigger.sh b/XB2Zarr/tools/trigger.sh new file mode 100644 index 0000000..9142dd5 --- /dev/null +++ b/XB2Zarr/tools/trigger.sh @@ -0,0 +1,3 @@ +curl -X POST http://localhost:3002/files/ \ + -F "xboutput=@test/xboutput_small.nc" \ + -F "id=test_xb" \ No newline at end of file From 7bc643f63fb9f3f3d49d4b11fbf413b20c3630bb Mon Sep 17 00:00:00 2001 From: Christopher J Lowrie Date: Thu, 4 Jan 2024 12:02:45 -0800 Subject: [PATCH 2/3] First commit, xb2zarr --- XB2Zarr/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/XB2Zarr/README.md b/XB2Zarr/README.md index e601ffc..b0a537e 100644 --- a/XB2Zarr/README.md +++ b/XB2Zarr/README.md @@ -1,6 +1,15 @@ ### Local Testing +``` ENV=dev BASE_GAR_DIRECTORY=us-west1-docker.pkg.dev/global-mangroves BASE_IMAGE=${BASE_GAR_DIRECTORY}/base/python_gis_base_${ENV} docker build -t xb2zarr --build-arg BASE_IMAGE=$BASE_IMAGE . -docker run -it -v $HOME/.config/gcloud:/root/.config/gcloud -e OUTPUT_BUCKET=geopmaker-output-staging -p 3002:8080 -v $PWD:/app xb2zarr \ No newline at end of file +docker run -it \ + --cap-add SYS_ADMIN --device /dev/fuse \ + -v $HOME/.config/gcloud:/root/.config/gcloud \ + -e MNT_BUCKETS="xbeach-outputs;xb2zarr" \ + -e OUTPUT_BUCKET=xb2zarr \ + -p 3002:8080 \ + -v $PWD:/app \ + xb2zarr +``` \ No newline at end of file From e2df2d4d95cb81f3154a14a29d404e461c5a0607 Mon Sep 17 00:00:00 2001 From: Christopher J Lowrie Date: Fri, 5 Jan 2024 09:32:20 -0800 Subject: [PATCH 3/3] Updates for combined NetCDF --- XB2Zarr/NetCDF_Multi_Function_Tool.py | 25 ++++++++- XB2Zarr/app.py | 76 +++++++++++++++++---------- XB2Zarr/tools/trigger.sh | 3 +- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/XB2Zarr/NetCDF_Multi_Function_Tool.py b/XB2Zarr/NetCDF_Multi_Function_Tool.py index 9f95461..df67ce0 100644 --- a/XB2Zarr/NetCDF_Multi_Function_Tool.py +++ b/XB2Zarr/NetCDF_Multi_Function_Tool.py @@ -27,8 +27,29 @@ def get_bounds(ds): - print(ds.isel(nx=0).globalx.min().compute()) - print(ds.isel(nx=-1).globalx.min().compute()) + return [ + ds.globalx.min().compute().values, + ds.globalx.max().compute().values, + ds.globaly.min().compute().values, + ds.globaly.max().compute().values + ] + +def transform_coords(ds, nx_max, ny_max, bounds): + xmin, xmax, ymin, ymax = bounds + new_x = [ + (xmin + i / ds.x.max() * (xmax - xmin)).values + for i in ds.x.values + ] + new_y = [ + (ymax - i / ds.y.max() * (ymax - ymin)).values + for i in ds.y.values + ] + ds = ds.assign_coords(x=new_x, y=new_y) + return ds + + + + def write_timestep_images(rds, full_output_path, start_time, end_time, variable, time_method): diff --git a/XB2Zarr/app.py b/XB2Zarr/app.py index c8f06f8..57dfbd4 100644 --- a/XB2Zarr/app.py +++ b/XB2Zarr/app.py @@ -6,7 +6,7 @@ import os, uvicorn, io import xarray as xr import rioxarray as rxr -from NetCDF_Multi_Function_Tool import write_timestep_images, get_bounds +from NetCDF_Multi_Function_Tool import write_timestep_images, get_bounds, transform_coords import uuid, shutil app = FastAPI() @@ -16,35 +16,55 @@ async def create_file( xboutput: Annotated[bytes, File()], id: Annotated[str, Form()], + vars: Annotated[str, Form()], ): xid = str(uuid.uuid1()) - output_dir = os.path.join("/tmp", xid) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - ds = xr.open_dataset(io.BytesIO(xboutput), chunks={"globaltime": 500}) - print(get_bounds(ds)) - return - write_timestep_images(ds, output_dir, 0, 2, "zs", "globaltime") - print(os.listdir(output_dir)) - print("file written, sending...") - if not os.path.exists("/results"): - os.makedirs("/results") - results_dir = f"/results/{xid}" - if not os.path.exists(results_dir): - os.makedirs(results_dir) - output_zarr = os.path.join(output_dir, f"{id}.zarr") - datasets = [ - rxr.open_rasterio(os.path.join(output_dir, i))\ - .expand_dims({"time": 1})\ - .assign_coords({"time": [int(i.split('.')[0].split('_')[-1])]}) - for i in os.listdir(output_dir) - ] - print(datasets[0]) - xr.concat( - datasets, - dim="time" - ).to_zarr(output_zarr) - shutil.make_archive(os.path.join(results_dir, "results"), "zip", output_dir) + vars = vars.split(',') + print(vars) + output_base_dir = os.path.join("/tmp", xid) + if not os.path.exists(output_base_dir): + os.makedirs(output_base_dir) + ds = xr.open_dataset(io.BytesIO(xboutput)) + bounds = get_bounds(ds) + nx_max = ds.nx.max() + ny_max = ds.ny.max() + outputs = [] + for var in vars: + print(var) + if var == "zs": + tdim="globaltime" + else: + tdim="meantime" + output_dir = os.path.join(output_base_dir, var) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + write_timestep_images(ds, output_dir, 0, 8, var, tdim) + if not os.path.exists("/results"): + os.makedirs("/results") + results_dir = f"/results/{xid}" + if not os.path.exists(results_dir): + os.makedirs(results_dir) + output_zarr = os.path.join(output_dir, f"{id}.zarr") + output_nc = os.path.join(output_dir, f"{id}.nc") + datasets = [ + transform_coords(rxr.open_rasterio(os.path.join(output_dir, i)), nx_max, ny_max, bounds).rio.write_crs(ds.attrs['crs']).rio.write_nodata(-1000)\ + .expand_dims({"time": 1})\ + .assign_coords({"time": [int(i.split('.')[0].split('_')[-1])]}) + for i in os.listdir(output_dir) + ] + # xr.concat( + # datasets, + # dim="time" + # ).to_zarr(output_zarr) + x = xr.concat( + datasets, + dim="time" + ).rename(var) + x.to_netcdf(output_nc) + outputs.append(x) + xr.merge(outputs).to_netcdf(os.path.join(output_base_dir, 'combined.nc')) + shutil.make_archive(os.path.join(results_dir, "results"), "zip", output_base_dir) return FileResponse(os.path.join(results_dir, "results.zip")) diff --git a/XB2Zarr/tools/trigger.sh b/XB2Zarr/tools/trigger.sh index 9142dd5..35d75dd 100644 --- a/XB2Zarr/tools/trigger.sh +++ b/XB2Zarr/tools/trigger.sh @@ -1,3 +1,4 @@ curl -X POST http://localhost:3002/files/ \ -F "xboutput=@test/xboutput_small.nc" \ - -F "id=test_xb" \ No newline at end of file + -F "id=test_xb" \ + -F "vars=zs,u_mean,v_mean" \ No newline at end of file