From a3d8c9dc8db96fc2f09ae80235b3f3cf31782134 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Fri, 7 Feb 2025 14:28:38 -0500
Subject: [PATCH 01/16] download script for inputs, merge layers initial task

---
 sample_data/tva.json             |  14 +-
 scripts/download_input_files.py  | 444 +++++++++++++++++++++++++++++++
 uvdat/core/tasks/merge_layers.py |  55 ++++
 3 files changed, 510 insertions(+), 3 deletions(-)
 create mode 100755 scripts/download_input_files.py
 create mode 100644 uvdat/core/tasks/merge_layers.py

diff --git a/sample_data/tva.json b/sample_data/tva.json
index 786dc1f..9d11019 100644
--- a/sample_data/tva.json
+++ b/sample_data/tva.json
@@ -62,7 +62,7 @@
     },
     {
         "type": "Context",
-        "name": "TVA Hydro Stations",
+        "name": "TVA Power Plants",
         "default_map_center": [
             34.8019,
             -86.1794
@@ -70,8 +70,8 @@
         "default_map_zoom": 6,
         "datasets": [
             {
-                "name": "TVA HydroStations",
-                "description": "TVA HydroStations",
+                "name": "TVA Power Plants",
+                "description": "TVA Power Plants",
                 "category": "vector",
                 "metadata": {},
                 "files": [
@@ -81,7 +81,15 @@
                         "name": "TVA Hydro Stations",
                         "type": "geojson",
                         "metadata": {}
+                    },
+                    {
+                        "path": "./data/tva/powerplants.zip",
+                        "url": "https://data.kitware.com/api/v1/file/67a645d7091a514e82eec881/download",
+                        "name": "TVA Power Plants",
+                        "type": "zip",
+                        "metadata": {}
                     }
+
                 ]
             }
         ]
diff --git a/scripts/download_input_files.py b/scripts/download_input_files.py
new file mode 100755
index 0000000..d99555a
--- /dev/null
+++ b/scripts/download_input_files.py
@@ -0,0 +1,444 @@
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+# Base URL pattern
+
+base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r11i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc"
+
+
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["585", "245", "126"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+# Base URL pattern
+
+base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r10i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc"
+
+
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["585", "245", "126"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+# Base URL pattern
+
+base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r4i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc"
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["585", "245", "126"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+base_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r4i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc"
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["126", "245", "585"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+base_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r5i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r5i1p1f2_gr_201501-210012.nc"
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["126", "245", "585"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r5i1p1f2_gr_201501-210012.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+import os
+
+import requests
+
+from IPython.display import display
+
+
+
+# Configuration
+
+output_dir = "/work/sds-lab/Puja/P-E/"
+
+
+
+ase_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r6i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc"
+
+
+
+variables = ["tas", "pr", "evspsbl"]
+
+scenarios = ["126", "245", "585"]
+
+
+
+# Function to download file
+
+def download_file(url, output_path):
+
+    try:
+
+        response = requests.get(url, stream=True)
+
+        response.raise_for_status()
+
+
+
+        with open(output_path, 'wb') as f:
+
+            for chunk in response.iter_content(chunk_size=8192):
+
+                f.write(chunk)
+
+
+
+        display(f"Downloaded: {output_path}")
+
+    except Exception as e:
+
+        display(f"Failed to download {url}: {e}")
+
+
+
+# Loop through all combinations
+
+for variable in variables:
+
+    for scenario in scenarios:
+
+        # Construct URL and output file path
+
+        file_url = base_url.format(scenario=scenario, variable=variable)
+
+        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc"
+
+        output_path = os.path.join(output_dir, file_name)
+
+
+
+        # Download the file
+
+        download_file(file_url, output_path)
+
+
+
diff --git a/uvdat/core/tasks/merge_layers.py b/uvdat/core/tasks/merge_layers.py
new file mode 100644
index 0000000..9f025b4
--- /dev/null
+++ b/uvdat/core/tasks/merge_layers.py
@@ -0,0 +1,55 @@
+from celery import shared_task
+from django.contrib.gis.geos import GEOSGeometry
+from uvdat.core.models import VectorMapLayer
+import json
+from typing import List, Optional, Literal
+
+
+@shared_task
+def merge_vector_layer_data(
+    base_layer_id: int,
+    other_layer_ids: List[int],
+    operation: Literal['intersection', 'complete_overlap'] = 'intersection',
+    exclude_non_overlapping: bool = False,
+    properties_to_merge: Optional[List[str]] = None
+) -> dict:
+    try:
+        base_layer = VectorMapLayer.objects.get(id=base_layer_id)
+        base_geojson = base_layer.read_geojson_data()
+        # Extract base features
+        base_features = base_geojson.get('features', [])
+        filtered_features = []
+        # Fetch specified vector map layers
+        other_layers = VectorMapLayer.objects.filter(id__in=other_layer_ids)
+        for base_feature in base_features:
+            base_geom = GEOSGeometry(json.dumps(base_feature['geometry']))
+            base_feature['overlapping_properties'] = []
+            has_overlap = False
+            for layer in other_layers:
+                layer_geojson = layer.read_geojson_data()
+                for feature in layer_geojson.get('features', []):
+                    feature_geom = GEOSGeometry(json.dumps(feature['geometry']))
+                    if operation == 'intersection' and base_geom.intersects(feature_geom):
+                        properties = feature['properties']
+                        if properties_to_merge:
+                            properties = {k: v for k, v in properties.items() if k in properties_to_merge}
+                        base_feature['overlapping_properties'].append(properties)
+                        has_overlap = True
+                    elif operation == 'complete_overlap' and base_geom.covers(feature_geom):
+                        properties = feature['properties']
+                        if properties_to_merge:
+                            properties = {k: v for k, v in properties.items() if k in properties_to_merge}
+                        base_feature['overlapping_properties'].append(properties)
+                        has_overlap = True
+            if not exclude_non_overlapping or has_overlap:
+                filtered_features.append(base_feature)
+
+        # Save the updated GeoJSON
+        updated_geojson = {'type': 'FeatureCollection', 'features': filtered_features}
+        base_layer.write_geojson_data(updated_geojson)
+        return {'status': 'success', 'message': 'Vector processing completed.'}
+
+    except VectorMapLayer.DoesNotExist:
+        return {'status': 'error', 'message': 'Base layer not found.'}
+    except Exception as e:
+        return {'status': 'error', 'message': str(e)}

From 360448b6401b90a5bb70dbc656288683e698ff8b Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Fri, 7 Feb 2025 14:30:30 -0500
Subject: [PATCH 02/16] update dev docs

---
 docs/dev-guide/dev.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/dev-guide/dev.md b/docs/dev-guide/dev.md
index 9bd2f35..fda19f0 100644
--- a/docs/dev-guide/dev.md
+++ b/docs/dev-guide/dev.md
@@ -10,7 +10,7 @@ To begin development, check out the repo via `git clone git@github.com:OpenGeosc
     2. Run `docker compose run --rm django ./manage.py createsuperuser`
        and follow the prompts to create your own user.
     3. Run `docker compose run --rm django ./manage.py makeclient --username {your.username@email.com}` to create the client application id for user logins.
-    4. Run `docker compose run --rm django ./manage.py ingest_data ./sample_data/test.json` to use sample data.
+    4. Run `docker compose run --rm django ./manage.py ingest_data tva.json` to use sample data.
     5. Copy the `./client/env.example` environment file to `./client/.env`.  The default environment variables should be sufficient for local development.
 
 ## Run Application

From 47580e711f2ef87b7f1e445a0cd3914ceb6b43c1 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 10 Feb 2025 12:19:00 -0500
Subject: [PATCH 03/16] scripts for merging data

---
 .../DataSelection/NetCDFDataConfigurator.vue  |   8 +-
 scripts/.gitignore                            |   4 +
 scripts/download_input_files.py               | 524 ++++--------------
 scripts/filter_geojson.py                     |  57 ++
 scripts/geojson_to_tsv.py                     |  40 ++
 scripts/merged_layers.py                      | 102 ++++
 uvdat/core/rest/__init__.py                   |   2 +
 uvdat/core/rest/tasks.py                      |  35 ++
 uvdat/core/tasks/merge_layers.py              | 157 ++++--
 uvdat/core/tasks/netcdf.py                    |   7 +-
 uvdat/urls.py                                 |   2 +
 11 files changed, 467 insertions(+), 471 deletions(-)
 create mode 100644 scripts/.gitignore
 create mode 100644 scripts/filter_geojson.py
 create mode 100644 scripts/geojson_to_tsv.py
 create mode 100644 scripts/merged_layers.py
 create mode 100644 uvdat/core/rest/tasks.py

diff --git a/client/src/components/DataSelection/NetCDFDataConfigurator.vue b/client/src/components/DataSelection/NetCDFDataConfigurator.vue
index 29a3d47..b98c36d 100644
--- a/client/src/components/DataSelection/NetCDFDataConfigurator.vue
+++ b/client/src/components/DataSelection/NetCDFDataConfigurator.vue
@@ -565,10 +565,10 @@ export default defineComponent({
                       <span> {{ modelValue.toFixed(2) }}</span>
                     </template>
                     <template #prepend>
-                      <span>{{ getVariableMinMax(newLayerX)[0] }}</span>
+                      <span>{{ getVariableMinMax(newLayerX)[0].toFixed(2) }}</span>
                     </template>
                     <template #append>
-                      <span>{{ getVariableMinMax(newLayerX)[1] }}</span>
+                      <span>{{ getVariableMinMax(newLayerX)[1].toFixed(2) }}</span>
                     </template>
                   </v-range-slider>
                 </div>
@@ -589,10 +589,10 @@ export default defineComponent({
                       <span> {{ modelValue.toFixed(2) }}</span>
                     </template>
                     <template #prepend>
-                      <span>{{ getVariableInformation(newLayerY)?.min }}</span>
+                      <span>{{ getVariableInformation(newLayerY)?.min.toFixed(2) }}</span>
                     </template>
                     <template #append>
-                      <span>{{ getVariableInformation(newLayerY)?.max }}</span>
+                      <span>{{ getVariableInformation(newLayerY)?.max.toFixed(2) }}</span>
                     </template>
                   </v-range-slider>
                   <v-tooltip v-if="getVariableInformation(newLayerY)?.geospatial === 'latitude'">
diff --git a/scripts/.gitignore b/scripts/.gitignore
new file mode 100644
index 0000000..0c6a867
--- /dev/null
+++ b/scripts/.gitignore
@@ -0,0 +1,4 @@
+**/*.json
+**/*.geojson
+**/*.csv
+**/*.tsv
\ No newline at end of file
diff --git a/scripts/download_input_files.py b/scripts/download_input_files.py
index d99555a..7ae0cd1 100755
--- a/scripts/download_input_files.py
+++ b/scripts/download_input_files.py
@@ -1,444 +1,120 @@
 import os
-
-import requests
-
-from IPython.display import display
-
-
-
-# Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-# Base URL pattern
-
-base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r11i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc"
-
-
-
-
-
-variables = ["tas", "pr", "evspsbl"]
-
-scenarios = ["585", "245", "126"]
-
-
-
-# Function to download file
-
-def download_file(url, output_path):
-
-    try:
-
-        response = requests.get(url, stream=True)
-
-        response.raise_for_status()
-
-
-
-        with open(output_path, 'wb') as f:
-
-            for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
-    except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc"
-
-        output_path = os.path.join(output_dir, file_name)
-
-
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-import os
-
-import requests
-
-from IPython.display import display
-
-
-
-# Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-# Base URL pattern
-
-base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r10i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc"
-
-
-
-
-
-variables = ["tas", "pr", "evspsbl"]
-
-scenarios = ["585", "245", "126"]
-
-
-
-# Function to download file
-
-def download_file(url, output_path):
-
-    try:
-
-        response = requests.get(url, stream=True)
-
-        response.raise_for_status()
-
-
-
-        with open(output_path, 'wb') as f:
-
-            for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
-    except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc"
-
-        output_path = os.path.join(output_dir, file_name)
-
-
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-import os
-
 import requests
-
-from IPython.display import display
-
-
+import click
+import xarray as xr
+import numpy as np
 
 # Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-# Base URL pattern
-
-base_url = "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r4i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc"
-
-
+default_output_dir = "./outputs"
+conus_dir = os.path.join(default_output_dir, "CONUS")
+
+# Base URL patterns and filenames
+base_urls = [
+    {
+        "url": "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r11i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc",
+        "filename": "{variable}_Amon_CESM2_ssp{scenario}_r11i1p1f1_gn_201501-206412.nc",
+    },
+    {
+        "url": "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r10i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc",
+        "filename": "{variable}_Amon_CESM2_ssp{scenario}_r10i1p1f1_gn_201501-206412.nc",
+    },
+    {
+        "url": "https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/NCAR/CESM2/ssp{scenario}/r4i1p1f1/Amon/{variable}/gn/v20200528/{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc",
+        "filename": "{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc",
+    },
+    {
+        "url": "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r4i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc",
+        "filename": "{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc",
+    },
+    {
+        "url": "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r6i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc",
+        "filename": "{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc",
+    },
+]
 
 variables = ["tas", "pr", "evspsbl"]
-
 scenarios = ["585", "245", "126"]
 
-
-
-# Function to download file
-
-def download_file(url, output_path):
-
-    try:
-
-        response = requests.get(url, stream=True)
-
-        response.raise_for_status()
-
-
-
-        with open(output_path, 'wb') as f:
-
-            for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
-    except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CESM2_ssp{scenario}_r4i1p1f1_gn_201501-206412.nc"
-
-        output_path = os.path.join(output_dir, file_name)
-
-
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-import os
-
-import requests
-
-from IPython.display import display
-
-
-
-# Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-base_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r4i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc"
-
-
-
-variables = ["tas", "pr", "evspsbl"]
-
-scenarios = ["126", "245", "585"]
-
-
-
-# Function to download file
-
-def download_file(url, output_path):
-
-    try:
-
-        response = requests.get(url, stream=True)
-
-        response.raise_for_status()
-
-
-
-        with open(output_path, 'wb') as f:
-
-            for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
-    except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r4i1p1f2_gr_201501-210012.nc"
-
-        output_path = os.path.join(output_dir, file_name)
-
-
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-import os
-
-import requests
-
-from IPython.display import display
-
-
-
-# Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-base_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r5i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r5i1p1f2_gr_201501-210012.nc"
-
-
-
-variables = ["tas", "pr", "evspsbl"]
-
-scenarios = ["126", "245", "585"]
-
-
-
-# Function to download file
-
-def download_file(url, output_path):
-
-    try:
-
-        response = requests.get(url, stream=True)
-
-        response.raise_for_status()
-
-
-
-        with open(output_path, 'wb') as f:
-
-            for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
-    except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r5i1p1f2_gr_201501-210012.nc"
-
-        output_path = os.path.join(output_dir, file_name)
-
-
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-import os
-
-import requests
-
-from IPython.display import display
-
-
-
-# Configuration
-
-output_dir = "/work/sds-lab/Puja/P-E/"
-
-
-
-ase_url = "https://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/ScenarioMIP/CNRM-CERFACS/CNRM-CM6-1/ssp{scenario}/r6i1p1f2/Amon/{variable}/gr/v20190410/{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc"
-
-
-
-variables = ["tas", "pr", "evspsbl"]
-
-scenarios = ["126", "245", "585"]
-
-
-
-# Function to download file
-
 def download_file(url, output_path):
+    """Download a file if it does not exist."""
+    if os.path.exists(output_path):
+        print(f"File already exists: {output_path}, skipping download.")
+        return True
 
     try:
-
         response = requests.get(url, stream=True)
-
         response.raise_for_status()
-
-
+        total_size = int(response.headers.get('content-length', 0))
+        downloaded = 0
 
         with open(output_path, 'wb') as f:
-
             for chunk in response.iter_content(chunk_size=8192):
-
-                f.write(chunk)
-
-
-
-        display(f"Downloaded: {output_path}")
-
+                if chunk:
+                    f.write(chunk)
+                    downloaded += len(chunk)
+                    progress = (downloaded / total_size) * 100 if total_size > 0 else 0
+                    print(f"Downloading {output_path} - {progress:.2f}% completed", end='\r')
+
+        print(f"\nDownloaded: {output_path}")
+        return True
     except Exception as e:
-
-        display(f"Failed to download {url}: {e}")
-
-
-
-# Loop through all combinations
-
-for variable in variables:
-
-    for scenario in scenarios:
-
-        # Construct URL and output file path
-
-        file_url = base_url.format(scenario=scenario, variable=variable)
-
-        file_name = f"{variable}_Amon_CNRM-CM6-1_ssp{scenario}_r6i1p1f2_gr_201501-210012.nc"
-
+        print(f"\nFailed to download {url}: {e}")
+        return False
+
+def process_conus(nc_path):
+    """Clip the netCDF data to a rectangle around the Continental U.S."""
+    if not os.path.exists(conus_dir):
+        os.makedirs(conus_dir, exist_ok=True)
+
+    output_conus_path = os.path.join(conus_dir, os.path.basename(nc_path))
+    if os.path.exists(output_conus_path):
+        print(f"CONUS-processed file already exists: {output_conus_path}, skipping.")
+        return
+
+    ds = xr.open_dataset(nc_path)
+
+    # Identify first variable with 'time', 'lat', and 'lon'
+    target_var = None
+    for var in ds.data_vars:
+        dims = ds[var].dims
+        if all(dim in dims for dim in ['time', 'lat', 'lon']):
+            target_var = var
+            break
+
+    if not target_var:
+        print(f"Skipping {nc_path}, no valid variable found.")
+        return
+
+    # Convert longitude from [0, 360] to [-180, 180]
+    ds = ds.assign_coords(lon=((ds.lon + 180) % 360) - 180)
+    ds = ds.sortby(ds.lon)
+
+    # Define CONUS bounds (approximate)
+    lat_min, lat_max = 20, 50
+    lon_min, lon_max = -130, -65
+    ds.sortby(ds.lat, True)
+    # Subset dataset for just the target variable
+    ds_conus = ds[target_var].sel(lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max))
+
+    # Save to new netCDF file
+    ds_conus.to_netcdf(output_conus_path)
+    print(f"Processed CONUS file saved: {output_conus_path}")
+
+
+
+@click.command()
+@click.option('--output_dir', default=default_output_dir, help='Directory to save downloaded files')
+def download_data(output_dir):
+    os.makedirs(output_dir, exist_ok=True)
+    file_list = [(entry, variable, scenario) for entry in base_urls for variable in variables for scenario in scenarios]
+
+    for entry, variable, scenario in file_list:
+        file_url = entry["url"].format(scenario=scenario, variable=variable)
+        file_name = entry["filename"].format(scenario=scenario, variable=variable)
         output_path = os.path.join(output_dir, file_name)
 
+        if download_file(file_url, output_path):
+            process_conus(output_path)
 
-
-        # Download the file
-
-        download_file(file_url, output_path)
-
-
-
+if __name__ == '__main__':
+    download_data()
diff --git a/scripts/filter_geojson.py b/scripts/filter_geojson.py
new file mode 100644
index 0000000..17430a1
--- /dev/null
+++ b/scripts/filter_geojson.py
@@ -0,0 +1,57 @@
+import click
+import geopandas as gpd
+import logging
+import json
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@click.command()
+@click.argument('geojson_file1', type=click.Path(exists=True))
+@click.argument('geojson_file2', type=click.Path(exists=True))
+@click.argument('output_file', type=click.Path())
+def filter_intersections(geojson_file1, geojson_file2, output_file):
+    """
+    A script to filter features from the first GeoJSON file that intersect with features
+    from the second GeoJSON file and save the filtered features into a new GeoJSON file.
+    """
+    # Load the GeoJSON files using GeoPandas
+    logger.info(f"Loading GeoJSON files: {geojson_file1}, {geojson_file2}")
+    gdf1 = gpd.read_file(geojson_file1)
+    gdf2 = gpd.read_file(geojson_file2)
+
+    # Ensure both datasets use the same coordinate reference system (CRS)
+    if gdf1.crs != gdf2.crs:
+        logger.info("CRS mismatch, reprojecting GeoJSON 2 to match GeoJSON 1.")
+        gdf2 = gdf2.to_crs(gdf1.crs)
+
+    # Prepare a list to store the intersecting features
+    logger.info("Finding intersecting features...")
+    filtered_features = []
+
+    # Loop through each feature in the first GeoDataFrame
+    for _, row1 in gdf1.iterrows():
+        # Check if the feature from gdf1 intersects with any feature from gdf2
+        intersecting_features = gdf2[gdf2.geometry.intersects(row1['geometry'])]
+
+        # If any intersection exists, add the feature to the filtered list
+        if not intersecting_features.empty:
+            # Convert the feature to GeoJSON format (including geometry)
+            feature_geojson = {
+                'type': 'Feature',
+                'geometry': row1['geometry'].__geo_interface__,
+                'properties': row1.drop('geometry').to_dict()
+            }
+            filtered_features.append(feature_geojson)
+
+    # Save the filtered features to a new GeoJSON file
+    logger.info(f"Saving filtered features to {output_file}")
+    with open(output_file, 'w') as f:
+        geojson = {"type": "FeatureCollection", "features": filtered_features}
+        json.dump(geojson, f)
+
+    logger.info("Filtering complete.")
+
+if __name__ == '__main__':
+    filter_intersections()
diff --git a/scripts/geojson_to_tsv.py b/scripts/geojson_to_tsv.py
new file mode 100644
index 0000000..7767f22
--- /dev/null
+++ b/scripts/geojson_to_tsv.py
@@ -0,0 +1,40 @@
+import click
+import geopandas as gpd
+import csv
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@click.command()
+@click.argument('geojson_file', type=click.Path(exists=True))
+@click.argument('output_tsv_file', type=click.Path())
+def geojson_to_tsv(geojson_file, output_tsv_file):
+    """
+    A script to convert a GeoJSON file into a TSV (Tab-Separated Values) file.
+    Each feature in the GeoJSON becomes a row in the TSV, 
+    with the feature properties as columns.
+    """
+    # Load GeoJSON file using GeoPandas
+    logger.info(f"Loading GeoJSON file: {geojson_file}")
+    gdf = gpd.read_file(geojson_file)
+
+    # Prepare the data for TSV (excluding geometry)
+    logger.info("Preparing the data for TSV conversion...")
+    data = gdf.drop(columns='geometry')  # Drop geometry to focus on properties
+
+    # Open the output TSV file and write the data
+    logger.info(f"Saving the result to {output_tsv_file}")
+    with open(output_tsv_file, 'w', newline='', encoding='utf-8') as f:
+        writer = csv.DictWriter(f, fieldnames=data.columns, delimiter='\t')  # Set delimiter to tab
+        writer.writeheader()  # Write the header row (columns)
+        
+        # Write each row as a dictionary
+        for _, row in data.iterrows():
+            writer.writerow(row.to_dict())
+
+    logger.info("Processing complete.")
+
+if __name__ == '__main__':
+    geojson_to_tsv()
diff --git a/scripts/merged_layers.py b/scripts/merged_layers.py
new file mode 100644
index 0000000..fc058ca
--- /dev/null
+++ b/scripts/merged_layers.py
@@ -0,0 +1,102 @@
+import click
+import geopandas as gpd
+import json
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@click.command()
+@click.argument('geojson_file1', type=click.Path(exists=True))
+@click.argument('geojson_files', type=click.Path(exists=True), nargs=-1)  # Accept multiple secondary files
+@click.argument('output_file', type=click.Path())
+@click.option('--flatten', default=True, is_flag=True, help="Flatten the properties of the intersecting features into a single string.")
+@click.option('--use_first_value', default=False, is_flag=True, help="Only take the first intersecting value for each property (do not flatten).")
+def intersect_geojson(geojson_file1, geojson_files, output_file, flatten, use_first_value):
+    """
+    A script to find all intersecting features from a primary GeoJSON file and a list of secondary GeoJSON files.
+    Each feature from the primary GeoJSON is converted into a GeoJSON feature,
+    and properties from the intersecting features in the secondary GeoJSON files are added into a 'merged_properties' field as a comma-separated string.
+    Only features with intersections are included in the output.
+    The properties from the secondary files are excluded from the root 'properties' and moved into 'merged_properties' as a joined string.
+    """
+    # Load the primary GeoJSON file using GeoPandas
+    logger.info(f"Loading primary GeoJSON file: {geojson_file1}")
+    gdf1 = gpd.read_file(geojson_file1)
+
+    # Prepare a list to hold the resulting features
+    output_features = []
+
+    # Loop through each secondary GeoJSON file
+    for geojson_file2 in geojson_files:
+        logger.info(f"Loading secondary GeoJSON file: {geojson_file2}")
+        gdf2 = gpd.read_file(geojson_file2)
+
+        # Ensure both datasets use the same coordinate reference system (CRS)
+        if gdf1.crs != gdf2.crs:
+            logger.info(f"CRS mismatch with {geojson_file2}, reprojecting.")
+            gdf2 = gdf2.to_crs(gdf1.crs)
+
+        # For each feature in the first GeoDataFrame, find intersecting features from the second GeoDataFrame
+        logger.info(f"Calculating intersects with {geojson_file2}...")
+        for _, row1 in gdf1.iterrows():
+            # Find features in gdf2 that intersect the current feature from gdf1
+            intersecting_features = gdf2[gdf2.geometry.intersects(row1['geometry'])]
+
+            # Only proceed if there are intersecting features
+            if not intersecting_features.empty:
+                # Prepare the properties dictionary (include all original properties from gdf1)
+                feature_properties = row1.drop('geometry')
+                for _, row_dict in intersecting_features.iterrows():
+                    for item in row_dict.to_dict().keys():
+                        if feature_properties.get(item):
+                            feature_properties = feature_properties.drop(item)
+                feature_properties = feature_properties.to_dict()
+
+                # Prepare 'merged_properties' and join intersecting feature properties as a comma-separated string
+                merged_properties = []
+                for _, row2 in intersecting_features.iterrows():
+                    merged_properties.append(row2.drop('geometry').to_dict())
+
+                if flatten and not use_first_value:
+                    new_vals = {}
+                    for i, prop in enumerate(merged_properties):
+                        for key, value in prop.items():
+                            if f'intersecting_{key}' not in new_vals:
+                                new_vals[f'intersecting_{key}'] = []
+                            new_vals[f'intersecting_{key}'].append(value)
+                    for key, value in new_vals.items():
+                        str_vals = [str(item) for item in value]
+                        feature_properties[key] = ', '.join(str_vals)
+                elif use_first_value:
+                    # If use_first_value is True, only take the first value for each intersecting property
+                    for prop in merged_properties:
+                        for key, value in prop.items():
+                            if f'intersecting_{key}' not in feature_properties:
+                                feature_properties[f'intersecting_{key}'] = value
+                    # No need to flatten or join as the first value is taken for each property
+
+                else:
+                    feature_properties['merged_properties'] = merged_properties
+
+                # Convert the feature to a GeoJSON feature
+                feature_geojson = {
+                    'type': 'Feature',
+                    'geometry': row1['geometry'].__geo_interface__,  # Convert geometry to GeoJSON format
+                    'properties': feature_properties  # Add properties to the GeoJSON feature
+                }
+
+                # Append the feature to the output list
+                output_features.append(feature_geojson)
+
+    # Save the output features as a new GeoJSON file
+    logger.info(f"Saving the result to {output_file}")
+    with open(output_file, 'w') as f:
+        geojson = {"type": "FeatureCollection", "features": output_features}
+        json.dump(geojson, f)
+
+    logger.info("Processing complete.")
+
+if __name__ == '__main__':
+    intersect_geojson()
diff --git a/uvdat/core/rest/__init__.py b/uvdat/core/rest/__init__.py
index b3c7705..a71f777 100644
--- a/uvdat/core/rest/__init__.py
+++ b/uvdat/core/rest/__init__.py
@@ -12,6 +12,7 @@
 from .simulations import SimulationViewSet
 from .user import UserViewSet
 from .vector_feature_table_data import VectorFeatureTableDataViewSet
+from .tasks import TasksAPIView
 
 __all__ = [
     ContextViewSet,
@@ -34,4 +35,5 @@
     ProcessingTaskView,
     UserViewSet,
     VectorFeatureTableDataViewSet,
+    TasksAPIView,
 ]
diff --git a/uvdat/core/rest/tasks.py b/uvdat/core/rest/tasks.py
new file mode 100644
index 0000000..12ad646
--- /dev/null
+++ b/uvdat/core/rest/tasks.py
@@ -0,0 +1,35 @@
+from rest_framework.viewsets import ViewSet
+from rest_framework.response import Response
+from rest_framework import status
+from rest_framework.decorators import action
+from drf_spectacular.utils import extend_schema
+from rest_framework.serializers import Serializer, IntegerField, CharField, BooleanField, ListField
+from uvdat.core.tasks.merge_layers import merge_vector_layer_data
+from .permissions import DefaultPermission
+
+class MergeVectorLayerSerializer(Serializer):
+    base_layer_id = IntegerField()
+    other_layer_ids = ListField(child=IntegerField())
+    dataset_name = CharField(max_length=255)
+    operation = CharField(default='intersection')
+    exclude_non_overlapping = BooleanField(default=True)
+    properties_to_merge = ListField(child=CharField(), required=False, allow_null=True)
+
+class TasksAPIView(ViewSet):
+    permission_classes = [DefaultPermission]
+
+    @extend_schema(request=MergeVectorLayerSerializer)
+    @action(detail=False, methods=['post'], url_path='merge-vector-layer')
+    def post(self, request, *args, **kwargs):
+        serializer = MergeVectorLayerSerializer(data=request.data)
+        if serializer.is_valid():
+            merge_vector_layer_data.delay(
+                base_layer_id=serializer.validated_data['base_layer_id'],
+                other_layer_ids=serializer.validated_data['other_layer_ids'],
+                dataset_name=serializer.validated_data['dataset_name'],
+                operation=serializer.validated_data['operation'],
+                exclude_non_overlapping=serializer.validated_data['exclude_non_overlapping'],
+                properties_to_merge=serializer.validated_data.get('properties_to_merge')
+            )
+            return Response({'status': 'success', 'message': 'Task has been initiated.'}, status=status.HTTP_202_ACCEPTED)
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
\ No newline at end of file
diff --git a/uvdat/core/tasks/merge_layers.py b/uvdat/core/tasks/merge_layers.py
index 9f025b4..ba19a93 100644
--- a/uvdat/core/tasks/merge_layers.py
+++ b/uvdat/core/tasks/merge_layers.py
@@ -1,55 +1,136 @@
+import logging
+import geopandas as gpd
+import pandas as pd
 from celery import shared_task
-from django.contrib.gis.geos import GEOSGeometry
-from uvdat.core.models import VectorMapLayer
+from uvdat.core.models import VectorMapLayer, Dataset, FileItem
 import json
 from typing import List, Optional, Literal
+from django.core.files.base import ContentFile
 
+# Configure logging
+logger = logging.getLogger(__name__)
+
+def save_geojson_to_dataset(dataset_name: str, geojson_data: dict):
+    dataset, _ = Dataset.objects.get_or_create(name=dataset_name, defaults={'category': 'generated'})
+    geojson_content = json.dumps(geojson_data, indent=2)
+    file_item = FileItem.objects.create(
+        name=f'{dataset_name}.geojson',
+        dataset=dataset,
+        file_type='geojson',
+        file_size=len(geojson_content.encode('utf-8')),
+    )
+    file_item.file.save(f'{dataset_name}.geojson', ContentFile(geojson_content.encode()), save=True)
+    return file_item
 
 @shared_task
 def merge_vector_layer_data(
     base_layer_id: int,
     other_layer_ids: List[int],
-    operation: Literal['intersection', 'complete_overlap'] = 'intersection',
-    exclude_non_overlapping: bool = False,
-    properties_to_merge: Optional[List[str]] = None
+    dataset_name: str,
+    operation: Literal['intersection', 'contains'] = 'intersection',
+    exclude_non_overlapping: bool = True,
+    properties_to_merge: Optional[List[str]] = None,
+    flatten: bool = True,  # Default to flattening
+    use_first_value: bool = False  # first value from merged properties
 ) -> dict:
+    logger.info(f'Starting merge_vector_layer_data task for base_layer_id: {base_layer_id}')
+
     try:
+        # Load the base layer GeoDataFrame
         base_layer = VectorMapLayer.objects.get(id=base_layer_id)
-        base_geojson = base_layer.read_geojson_data()
-        # Extract base features
-        base_features = base_geojson.get('features', [])
-        filtered_features = []
-        # Fetch specified vector map layers
-        other_layers = VectorMapLayer.objects.filter(id__in=other_layer_ids)
-        for base_feature in base_features:
-            base_geom = GEOSGeometry(json.dumps(base_feature['geometry']))
-            base_feature['overlapping_properties'] = []
-            has_overlap = False
-            for layer in other_layers:
-                layer_geojson = layer.read_geojson_data()
-                for feature in layer_geojson.get('features', []):
-                    feature_geom = GEOSGeometry(json.dumps(feature['geometry']))
-                    if operation == 'intersection' and base_geom.intersects(feature_geom):
-                        properties = feature['properties']
-                        if properties_to_merge:
-                            properties = {k: v for k, v in properties.items() if k in properties_to_merge}
-                        base_feature['overlapping_properties'].append(properties)
-                        has_overlap = True
-                    elif operation == 'complete_overlap' and base_geom.covers(feature_geom):
-                        properties = feature['properties']
-                        if properties_to_merge:
-                            properties = {k: v for k, v in properties.items() if k in properties_to_merge}
-                        base_feature['overlapping_properties'].append(properties)
-                        has_overlap = True
-            if not exclude_non_overlapping or has_overlap:
-                filtered_features.append(base_feature)
-
-        # Save the updated GeoJSON
-        updated_geojson = {'type': 'FeatureCollection', 'features': filtered_features}
-        base_layer.write_geojson_data(updated_geojson)
-        return {'status': 'success', 'message': 'Vector processing completed.'}
+        base_gdf = gpd.GeoDataFrame.from_features(base_layer.read_geojson_data().get('features', []))
+
+        # Load the secondary layers and store their properties
+        other_gdfs = []
+        for layer_id in other_layer_ids:
+            layer = VectorMapLayer.objects.get(id=layer_id)
+            layer_gdf = gpd.GeoDataFrame.from_features(layer.read_geojson_data().get('features', []))
+            other_gdfs.append(layer_gdf)
+
+        if not other_gdfs:
+            logger.warning('No valid other layers found. Skipping merge.')
+            return {'status': 'error', 'message': 'No valid other layers provided.'}
+
+        # Merge all secondary layers into a single GeoDataFrame
+        other_gdf = pd.concat(other_gdfs, ignore_index=True)
+
+        # Ensure both layers use the same CRS
+        if base_gdf.crs != other_gdf.crs:
+            logger.info('Reprojecting other layers to match base layer CRS...')
+            other_gdf = other_gdf.to_crs(base_gdf.crs)
+
+        # Prepare list to collect the merged features
+        output_features = []
+
+        # Loop through each feature in the base layer
+        logger.info(f"Processing {len(base_gdf)} features from the base layer.")
+        for _, base_row in base_gdf.iterrows():
+            # Find features in the other layers that intersect or completely overlap the current base layer feature
+            intersecting_features = []
+            for _, other_row in other_gdf.iterrows():
+                if operation == 'intersection' and base_row['geometry'].intersects(other_row['geometry']):
+                    intersecting_features.append(other_row)
+                elif operation == 'contains' and base_row['geometry'].contains(other_row['geometry']):
+                    intersecting_features.append(other_row)
+
+            # If exclude_non_overlapping is True and no intersection or complete overlap is found, skip this base feature
+            if exclude_non_overlapping and not intersecting_features:
+                continue
+
+            # Prepare the properties for the current feature
+            feature_properties = base_row.drop('geometry')
+
+            # Merge properties from the intersecting features
+            merged_properties = []
+            for other_row in intersecting_features:
+                other_props = other_row.drop('geometry').to_dict()
+
+                # If properties_to_merge is provided, only merge those
+                if properties_to_merge:
+                    filtered_props = {key: value for key, value in other_props.items() if key in properties_to_merge}
+                    merged_properties.append(filtered_props)
+                else:
+                    merged_properties.append(other_props)
+
+            # Flatten or use first intersecting value for properties
+            if flatten:
+                new_vals = {}
+                for prop in merged_properties:
+                    for key, value in prop.items():
+                        if f'intersecting_{key}' not in new_vals:
+                            new_vals[f'intersecting_{key}'] = []
+                        new_vals[f'intersecting_{key}'].append(value)
+
+                if use_first_value:
+                    for key, value in feature_properties.items():
+                        if isinstance(value, list):
+                            feature_properties[key] = value[0]
+                else:
+                    for key, value in new_vals.items():
+                        feature_properties[key] = ', '.join(map(str, value))
+            else:
+                feature_properties['merged_properties'] = merged_properties
+
+            # Convert the feature to a GeoJSON feature
+            feature_geojson = {
+                'type': 'Feature',
+                'geometry': base_row['geometry'].__geo_interface__,  # Convert geometry to GeoJSON format
+                'properties': feature_properties  # Add properties to the GeoJSON feature
+            }
+
+            # Append the feature to the output list
+            output_features.append(feature_geojson)
+
+        # Save the output features as a new GeoJSON file
+        logger.info(f"Saving the result to {dataset_name}.geojson")
+        geojson = {"type": "FeatureCollection", "features": output_features}
+        save_geojson_to_dataset(dataset_name, geojson)
+
+        return {'status': 'success', 'message': 'Vector processing completed and dataset created.'}
 
     except VectorMapLayer.DoesNotExist:
+        logger.error(f'Base layer with ID {base_layer_id} not found.')
         return {'status': 'error', 'message': 'Base layer not found.'}
     except Exception as e:
+        logger.error(f'Error in merge_vector_layer_data: {str(e)}')
         return {'status': 'error', 'message': str(e)}
diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index 1de1867..fc2cc40 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -248,7 +248,7 @@ def preview_netcdf_slice(
             except Exception as e:
                 logger.warning(f'Slicer Range Exception: {e}')
                 slicer_range = None
-
+        ds = ds.sortby(ds[y_variable], ascending=False)
         data_var = ds.get(variable)
         variables = data_var.dims
         base_variables = (x_variable, y_variable, sliding_variable)
@@ -288,8 +288,6 @@ def preview_netcdf_slice(
         # Convert to an RGB image using PIL
         image = Image.fromarray(colored_data, mode='RGB')
         image_buffer = BytesIO()
-        if longitude360:
-            image = image.transpose(Image.FLIP_TOP_BOTTOM)
         image.save(image_buffer, format='PNG')
         image_buffer.seek(0)
         base64_image = base64.b64encode(image_buffer.getvalue()).decode('utf-8')
@@ -478,6 +476,7 @@ def create_netcdf_slices(
                 logger.warning(f'Slicer Range Exception: {e}')
                 slicer_range = None
         # Extract the data for the specified variable
+        ds = ds.sortby(ds[y_variable], ascending=False)
         data_var = ds.get(variable)
         variables_data = data_var.dims
         dim_size = ds.dims.get(sliding_variable)
@@ -640,8 +639,6 @@ def create_netcdf_slices(
             # Convert to an RGB image using PIL
             image = Image.fromarray(colored_data, mode='RGB')
             image_buffer = BytesIO()
-            if longitude360:
-                image = image.transpose(Image.FLIP_TOP_BOTTOM)
             image.save(image_buffer, format='PNG')
             image_buffer.seek(0)
             image_name = f'{variable}_{sliding_variable}_{i}.png'
diff --git a/uvdat/urls.py b/uvdat/urls.py
index cf8898a..8985291 100644
--- a/uvdat/urls.py
+++ b/uvdat/urls.py
@@ -25,6 +25,7 @@
     UserViewSet,
     VectorFeatureTableDataViewSet,
     VectorMapLayerViewSet,
+    TasksAPIView,
 )
 
 router = routers.SimpleRouter()
@@ -58,6 +59,7 @@
 router.register(r'netcdf', NetCDFDataView, basename='netcdf')
 router.register(r'processing-tasks', ProcessingTaskView, basename='processing-tasks')
 router.register(r'users', UserViewSet, basename='users')
+router.register(r'tasks', TasksAPIView, basename='tasks')
 
 urlpatterns = [
     path('accounts/', include('allauth.urls')),

From 5a5fc0a124696637b5c227aa05f778f24f9cc301 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 10 Feb 2025 12:45:07 -0500
Subject: [PATCH 04/16] linting, map_merge fixing, loading selection settings

---
 client/src/map/mapLayers.ts      |  1 +
 uvdat/core/rest/__init__.py      |  2 +-
 uvdat/core/rest/tasks.py         | 21 ++++++----
 uvdat/core/tasks/merge_layers.py | 70 ++++++++++++++++++++++++--------
 uvdat/urls.py                    |  2 +-
 5 files changed, 69 insertions(+), 27 deletions(-)

diff --git a/client/src/map/mapLayers.ts b/client/src/map/mapLayers.ts
index 68497a7..ed20590 100644
--- a/client/src/map/mapLayers.ts
+++ b/client/src/map/mapLayers.ts
@@ -106,6 +106,7 @@ const toggleLayerSelection = (layer: VectorMapLayer | RasterMapLayer | NetCDFLay
   } else {
     MapStore.selectedMapLayers.value = [...MapStore.selectedMapLayers.value, layer];
     toggleLayerVisibility(layer, true);
+    setPopupEvents(internalMap.value as maplibregl.Map);
   }
 };
 
diff --git a/uvdat/core/rest/__init__.py b/uvdat/core/rest/__init__.py
index a71f777..e5a372f 100644
--- a/uvdat/core/rest/__init__.py
+++ b/uvdat/core/rest/__init__.py
@@ -10,9 +10,9 @@
 from .processing_task import ProcessingTaskView
 from .regions import DerivedRegionViewSet, SourceRegionViewSet
 from .simulations import SimulationViewSet
+from .tasks import TasksAPIView
 from .user import UserViewSet
 from .vector_feature_table_data import VectorFeatureTableDataViewSet
-from .tasks import TasksAPIView
 
 __all__ = [
     ContextViewSet,
diff --git a/uvdat/core/rest/tasks.py b/uvdat/core/rest/tasks.py
index 12ad646..9329c66 100644
--- a/uvdat/core/rest/tasks.py
+++ b/uvdat/core/rest/tasks.py
@@ -1,12 +1,15 @@
-from rest_framework.viewsets import ViewSet
-from rest_framework.response import Response
+from drf_spectacular.utils import extend_schema
 from rest_framework import status
 from rest_framework.decorators import action
-from drf_spectacular.utils import extend_schema
-from rest_framework.serializers import Serializer, IntegerField, CharField, BooleanField, ListField
+from rest_framework.response import Response
+from rest_framework.serializers import BooleanField, CharField, IntegerField, ListField, Serializer
+from rest_framework.viewsets import ViewSet
+
 from uvdat.core.tasks.merge_layers import merge_vector_layer_data
+
 from .permissions import DefaultPermission
 
+
 class MergeVectorLayerSerializer(Serializer):
     base_layer_id = IntegerField()
     other_layer_ids = ListField(child=IntegerField())
@@ -15,6 +18,7 @@ class MergeVectorLayerSerializer(Serializer):
     exclude_non_overlapping = BooleanField(default=True)
     properties_to_merge = ListField(child=CharField(), required=False, allow_null=True)
 
+
 class TasksAPIView(ViewSet):
     permission_classes = [DefaultPermission]
 
@@ -29,7 +33,10 @@ def post(self, request, *args, **kwargs):
                 dataset_name=serializer.validated_data['dataset_name'],
                 operation=serializer.validated_data['operation'],
                 exclude_non_overlapping=serializer.validated_data['exclude_non_overlapping'],
-                properties_to_merge=serializer.validated_data.get('properties_to_merge')
+                properties_to_merge=serializer.validated_data.get('properties_to_merge'),
+            )
+            return Response(
+                {'status': 'success', 'message': 'Task has been initiated.'},
+                status=status.HTTP_202_ACCEPTED,
             )
-            return Response({'status': 'success', 'message': 'Task has been initiated.'}, status=status.HTTP_202_ACCEPTED)
-        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
\ No newline at end of file
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
diff --git a/uvdat/core/tasks/merge_layers.py b/uvdat/core/tasks/merge_layers.py
index ba19a93..e3bfb2b 100644
--- a/uvdat/core/tasks/merge_layers.py
+++ b/uvdat/core/tasks/merge_layers.py
@@ -1,27 +1,42 @@
+import json
 import logging
-import geopandas as gpd
-import pandas as pd
+from typing import List, Literal, Optional
+
 from celery import shared_task
-from uvdat.core.models import VectorMapLayer, Dataset, FileItem
-import json
-from typing import List, Optional, Literal
 from django.core.files.base import ContentFile
+import geopandas as gpd
+import pandas as pd
+
+from uvdat.core.models import Dataset, FileItem, VectorMapLayer
+
+from .dataset import process_file_item
 
 # Configure logging
 logger = logging.getLogger(__name__)
 
+
 def save_geojson_to_dataset(dataset_name: str, geojson_data: dict):
-    dataset, _ = Dataset.objects.get_or_create(name=dataset_name, defaults={'category': 'generated'})
+    dataset, _ = Dataset.objects.get_or_create(
+        name=dataset_name,
+        defaults={
+            'category': 'generated',
+            'description': 'Generated dataset from vector layer merge.',
+            'metadata': {},
+        },
+    )
     geojson_content = json.dumps(geojson_data, indent=2)
     file_item = FileItem.objects.create(
         name=f'{dataset_name}.geojson',
         dataset=dataset,
         file_type='geojson',
+        metadata={},
         file_size=len(geojson_content.encode('utf-8')),
     )
     file_item.file.save(f'{dataset_name}.geojson', ContentFile(geojson_content.encode()), save=True)
+    process_file_item.delay(file_item.id)
     return file_item
 
+
 @shared_task
 def merge_vector_layer_data(
     base_layer_id: int,
@@ -31,20 +46,24 @@ def merge_vector_layer_data(
     exclude_non_overlapping: bool = True,
     properties_to_merge: Optional[List[str]] = None,
     flatten: bool = True,  # Default to flattening
-    use_first_value: bool = False  # first value from merged properties
+    use_first_value: bool = False,  # first value from merged properties
 ) -> dict:
     logger.info(f'Starting merge_vector_layer_data task for base_layer_id: {base_layer_id}')
 
     try:
         # Load the base layer GeoDataFrame
         base_layer = VectorMapLayer.objects.get(id=base_layer_id)
-        base_gdf = gpd.GeoDataFrame.from_features(base_layer.read_geojson_data().get('features', []))
+        base_gdf = gpd.GeoDataFrame.from_features(
+            base_layer.read_geojson_data().get('features', [])
+        )
 
         # Load the secondary layers and store their properties
         other_gdfs = []
         for layer_id in other_layer_ids:
             layer = VectorMapLayer.objects.get(id=layer_id)
-            layer_gdf = gpd.GeoDataFrame.from_features(layer.read_geojson_data().get('features', []))
+            layer_gdf = gpd.GeoDataFrame.from_features(
+                layer.read_geojson_data().get('features', [])
+            )
             other_gdfs.append(layer_gdf)
 
         if not other_gdfs:
@@ -63,23 +82,32 @@ def merge_vector_layer_data(
         output_features = []
 
         # Loop through each feature in the base layer
-        logger.info(f"Processing {len(base_gdf)} features from the base layer.")
+        logger.info(f'Processing {len(base_gdf)} features from the base layer.')
         for _, base_row in base_gdf.iterrows():
             # Find features in the other layers that intersect or completely overlap the current base layer feature
             intersecting_features = []
             for _, other_row in other_gdf.iterrows():
-                if operation == 'intersection' and base_row['geometry'].intersects(other_row['geometry']):
+                if operation == 'intersection' and base_row['geometry'].intersects(
+                    other_row['geometry']
+                ):
                     intersecting_features.append(other_row)
-                elif operation == 'contains' and base_row['geometry'].contains(other_row['geometry']):
+                elif operation == 'contains' and base_row['geometry'].contains(
+                    other_row['geometry']
+                ):
                     intersecting_features.append(other_row)
 
-            # If exclude_non_overlapping is True and no intersection or complete overlap is found, skip this base feature
             if exclude_non_overlapping and not intersecting_features:
                 continue
 
             # Prepare the properties for the current feature
             feature_properties = base_row.drop('geometry')
 
+            for row_dict in intersecting_features:
+                for item in row_dict.to_dict().keys():
+                    if feature_properties.get(item):
+                        feature_properties = feature_properties.drop(item)
+            feature_properties = feature_properties.to_dict()
+
             # Merge properties from the intersecting features
             merged_properties = []
             for other_row in intersecting_features:
@@ -87,7 +115,11 @@ def merge_vector_layer_data(
 
                 # If properties_to_merge is provided, only merge those
                 if properties_to_merge:
-                    filtered_props = {key: value for key, value in other_props.items() if key in properties_to_merge}
+                    filtered_props = {
+                        key: value
+                        for key, value in other_props.items()
+                        if key in properties_to_merge
+                    }
                     merged_properties.append(filtered_props)
                 else:
                     merged_properties.append(other_props)
@@ -114,16 +146,18 @@ def merge_vector_layer_data(
             # Convert the feature to a GeoJSON feature
             feature_geojson = {
                 'type': 'Feature',
-                'geometry': base_row['geometry'].__geo_interface__,  # Convert geometry to GeoJSON format
-                'properties': feature_properties  # Add properties to the GeoJSON feature
+                'geometry': base_row[
+                    'geometry'
+                ].__geo_interface__,  # Convert geometry to GeoJSON format
+                'properties': feature_properties,  # Add properties to the GeoJSON feature
             }
 
             # Append the feature to the output list
             output_features.append(feature_geojson)
 
         # Save the output features as a new GeoJSON file
-        logger.info(f"Saving the result to {dataset_name}.geojson")
-        geojson = {"type": "FeatureCollection", "features": output_features}
+        logger.info(f'Saving the result to {dataset_name}.geojson')
+        geojson = {'type': 'FeatureCollection', 'features': output_features}
         save_geojson_to_dataset(dataset_name, geojson)
 
         return {'status': 'success', 'message': 'Vector processing completed and dataset created.'}
diff --git a/uvdat/urls.py b/uvdat/urls.py
index 8985291..e282ed7 100644
--- a/uvdat/urls.py
+++ b/uvdat/urls.py
@@ -22,10 +22,10 @@
     RasterMapLayerViewSet,
     SimulationViewSet,
     SourceRegionViewSet,
+    TasksAPIView,
     UserViewSet,
     VectorFeatureTableDataViewSet,
     VectorMapLayerViewSet,
-    TasksAPIView,
 )
 
 router = routers.SimpleRouter()

From 09b636517066e9db86aaf5e7ff19952c7f82e1a8 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 10 Feb 2025 12:56:57 -0500
Subject: [PATCH 05/16] renaming NetCDFLayers

---
 uvdat/core/rest/map_layers.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/uvdat/core/rest/map_layers.py b/uvdat/core/rest/map_layers.py
index e683f3a..15f78a0 100644
--- a/uvdat/core/rest/map_layers.py
+++ b/uvdat/core/rest/map_layers.py
@@ -14,6 +14,7 @@
 from uvdat.core.models import (
     LayerRepresentation,
     NetCDFLayer,
+    NetCDFData,
     RasterMapLayer,
     VectorFeature,
     VectorMapLayer,
@@ -580,6 +581,8 @@ def update_name(self, request, *args, **kwargs):
                 VectorMapLayer.objects.filter(id=layer_id).update(name=new_name)
             elif layer_type == 'raster':
                 RasterMapLayer.objects.filter(id=layer_id).update(name=new_name)
+            elif layer_type == 'netcdf':
+                NetCDFData.objects.filter(id=layer_id).update(name=new_name)
             else:
                 return Response(
                     {'error': 'Invalid layer type. Must be "vector" or "raster".'},

From 16fd490ff4db22dd915a3ae517f056544630f332 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 10 Feb 2025 14:55:31 -0500
Subject: [PATCH 06/16] add time mapping to netcdf layers

---
 client/src/components/NetCDFLayerConfig.vue | 35 ++++++++++-
 client/src/types.ts                         |  1 +
 scripts/netcdf_explorer.py                  | 31 +++++++---
 uvdat/core/rest/netcdf.py                   | 10 ++++
 uvdat/core/tasks/netcdf.py                  | 65 +++++++++++++++------
 5 files changed, 114 insertions(+), 28 deletions(-)

diff --git a/client/src/components/NetCDFLayerConfig.vue b/client/src/components/NetCDFLayerConfig.vue
index 28972df..2c5be99 100644
--- a/client/src/components/NetCDFLayerConfig.vue
+++ b/client/src/components/NetCDFLayerConfig.vue
@@ -27,6 +27,36 @@ export default defineComponent({
       const found = visibleNetCDFLayers.find((item) => item.netCDFLayer === props.layer.id);
       return found?.images.length || 0;
     });
+    const stepMapping = computed(() => {
+      const found = visibleNetCDFLayers.find((item) => item.netCDFLayer === props.layer.id);
+      const mapSlicer: Record<number, string | number> = {};
+      let unixTimeStamp = true;
+      if (found) {
+        if (found?.sliding) {
+          for (let i = 0; i < found.images.length; i += 1) {
+            mapSlicer[i] = found.sliding.min + i * found.sliding.step;
+            if (found.sliding.variable === 'time') {
+              // convert unix timestamp to human readable date YYYY-MM-DD
+              try {
+                const date = new Date((mapSlicer[i] as number) / 1e6);
+                // eslint-disable-next-line prefer-destructuring
+                mapSlicer[i] = date.toISOString().split('T')[0];
+              } catch (e) {
+                unixTimeStamp = false;
+                break;
+              }
+            }
+          }
+          if (unixTimeStamp) {
+            return mapSlicer;
+          }
+        }
+        for (let i = 0; i < found.images.length; i += 1) {
+          mapSlicer[i] = i;
+        }
+      }
+      return mapSlicer;
+    });
     const updateIndex = () => {
       updateNetCDFLayer(props.layer.id, currentIndex.value);
     };
@@ -43,6 +73,7 @@ export default defineComponent({
       layerOpacity,
       currentIndex,
       totalIndex,
+      stepMapping,
     };
   },
 });
@@ -72,9 +103,9 @@ export default defineComponent({
         <v-icon>
           mdi-checkbox-marked
         </v-icon>
-        <span class="pl-2">Step
+        <span class="pl-2">
           {{
-            currentIndex
+            stepMapping[currentIndex]
           }}</span>
       </v-col>
       <v-col>
diff --git a/client/src/types.ts b/client/src/types.ts
index f824266..63d8dda 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -441,6 +441,7 @@ export interface NetCDFImages {
   netCDFLayer: number;
   parent_bounds: [[[number, number], [number, number], [number, number], [number, number], [number, number]]];
   images: string[];
+  sliding: { min: number, max: number; step: number, variable: string };
 }
 
 export interface NetCDFVariable {
diff --git a/scripts/netcdf_explorer.py b/scripts/netcdf_explorer.py
index 3f55e91..e5adcf3 100644
--- a/scripts/netcdf_explorer.py
+++ b/scripts/netcdf_explorer.py
@@ -1,5 +1,8 @@
 import click
+import cftime
+import pandas as pd
 import xarray as xr
+import datetime
 from PIL import Image
 import numpy as np
 import os
@@ -37,16 +40,30 @@ def describe(file_path, output_json):
                 "dtype": str(variable.dtype),
                 "attributes": {key: str(value) for key, value in variable.attrs.items()},
             }
-
             # Calculate min and max values if the variable has numeric data
-            try:
-                var_min = float(variable.min().values) if variable.size > 0 else None
-                var_max = float(variable.max().values) if variable.size > 0 else None
+            if isinstance(variable.values[0], cftime.DatetimeNoLeap):
+                vals = []
+                for item in variable.values:
+                    print(item)
+                    dt = item
+                    dt_obj = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+                    unix_timestamp = pd.Timestamp(dt_obj).timestamp()
+                    vals.append(unix_timestamp)
+                print(f'TimeStamp: {unix_timestamp}')
+                var_min = float(min(vals)) if variable.size > 0 else None
+                var_max = float(max(vals)) if variable.size > 0 else None
                 var_info["min"] = var_min
                 var_info["max"] = var_max
-            except Exception:
-                var_info["min"] = 0
-                var_info["max"] = variable.size
+            else:
+                try:
+                    var_min = float(variable.min().values) if variable.size > 0 else None
+                    var_max = float(variable.max().values) if variable.size > 0 else None
+                    var_info["min"] = var_min
+                    var_info["max"] = var_max
+                except Exception:
+                    var_info["min"] = 0
+                    var_info["max"] = variable.size
+                    var_info["steps"] = variable.size
 
             description["variables"][var_name] = var_info
 
diff --git a/uvdat/core/rest/netcdf.py b/uvdat/core/rest/netcdf.py
index 0941e4f..267824f 100644
--- a/uvdat/core/rest/netcdf.py
+++ b/uvdat/core/rest/netcdf.py
@@ -80,9 +80,19 @@ def images(self, request, *args, **kwargs):
             )
 
             # Prepare the response data with image URLs and parent bounds
+            sliding_dim = netcdf_layer.parameters.get('sliding_dimension', None)
+            step_count = netcdf_layer.parameters.get('stepCount', None)
+            if sliding_dim and step_count:
+                sliding_data = {
+                    "min": sliding_dim.get('min', 0),
+                    "max": sliding_dim.get('max', step_count),
+                    "variable": sliding_dim.get('variable', 'time'),
+                }
+                sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / step_count
             response_data = {
                 'netCDFLayer': int(netcdf_layer_id),
                 'parent_bounds': netcdf_layer.bounds.envelope.coords,  # 4-tuple for the parent layer bounds
+                "sliding": sliding_data,
                 'images': [image.image.url for image in netcdf_images],  # Only return the image URL
             }
 
diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index fc2cc40..8f33586 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -1,10 +1,12 @@
 import base64
 from datetime import datetime
+import cftime
 from io import BytesIO
 import logging
 from pathlib import Path
 import re
 import tempfile
+import pandas as pd
 
 from PIL import Image
 from celery import current_task, shared_task
@@ -47,27 +49,52 @@ def create_netcdf_data_layer(file_item, metadata):
             }
 
             # Calculate min and max values if the variable has numeric data
-            try:
-                var_min = float(variable.min().values) if variable.size > 0 else None
-                var_max = float(variable.max().values) if variable.size > 0 else None
-                if 'datetime' in str(variable.dtype):
-                    var_info['startDate'] = str(variable.min().values)
-                    var_info['endDate'] = str(variable.max().values)
+            if isinstance(variable.values[0], (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+                vals = []
+                str_vals = []
+                for item in variable.values:
+                    print(item)
+                    dt = item
+                    dt_obj = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+                    unix_timestamp = pd.Timestamp(dt_obj).timestamp()
+                    str_vals.append(pd.Timestamp(dt_obj).isoformat())
+                    vals.append(unix_timestamp)
+                var_min = float(min(vals)) if variable.size > 0 else None
+                var_max = float(max(vals)) if variable.size > 0 else None
                 var_info['min'] = var_min
                 var_info['max'] = var_max
-                if var_name in description['dimensions'].keys():
-                    var_info['steps'] = description['dimensions'][var_name]
-                if re.search(r'\blat\b|\blatitude\b', var_name, re.IGNORECASE):
-                    if -90 <= var_min <= 90 and -90 <= var_max <= 90:
-                        var_info['geospatial'] = 'latitude'
-                elif re.search(r'\blon\b|\blongitude\b', var_name, re.IGNORECASE):
-                    if -180 <= var_min <= 180 and -180 <= var_max <= 180:
-                        var_info['geospatial'] = 'longitude'
-                    elif 0 <= var_min <= 360 and 0 <= var_max <= 360:
-                        var_info['geospatial'] = 'longitude360'
-            except Exception:
-                var_info['min'] = 0
-                var_info['max'] = variable.size
+                var_info['steps'] = variable.size
+                # var_info['timeMap'] = str_vals
+            else:
+                try:
+                    var_min = float(variable.min().values) if variable.size > 0 else None
+                    var_max = float(variable.max().values) if variable.size > 0 else None
+                    if 'datetime' in str(variable.dtype):
+                        var_info['startDate'] = str(variable.min().values)
+                        var_info['endDate'] = str(variable.max().values)
+                        # str_vals = []
+                        # for item in variable.values:
+                        #     str_vals.append(str(item))
+                        #     var_info['timeMap'] = str_vals
+
+                    var_info['min'] = var_min
+                    var_info['max'] = var_max
+                    var_info['steps'] = variable.size
+
+                    if var_name in description['dimensions'].keys():
+                        var_info['steps'] = description['dimensions'][var_name]
+                    if re.search(r'\blat\b|\blatitude\b', var_name, re.IGNORECASE):
+                        if -90 <= var_min <= 90 and -90 <= var_max <= 90:
+                            var_info['geospatial'] = 'latitude'
+                    elif re.search(r'\blon\b|\blongitude\b', var_name, re.IGNORECASE):
+                        if -180 <= var_min <= 180 and -180 <= var_max <= 180:
+                            var_info['geospatial'] = 'longitude'
+                        elif 0 <= var_min <= 360 and 0 <= var_max <= 360:
+                            var_info['geospatial'] = 'longitude360'
+                except Exception:
+                    var_info['min'] = 0
+                    var_info['max'] = variable.size
+                    var_info["steps"] = variable.size
 
             description['variables'][var_name] = var_info
 

From 59b13896579ed52a83869e07806f720f7f7c98e3 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Tue, 11 Feb 2025 08:58:11 -0500
Subject: [PATCH 07/16] adjusting timestamps to unix base

---
 .../DataSelection/NetCDFDataConfigurator.vue  | 16 +--
 client/src/components/NetCDFLayerConfig.vue   |  2 +-
 client/src/utils.ts                           | 60 +++++++++++-
 uvdat/core/rest/netcdf.py                     |  2 +-
 uvdat/core/tasks/netcdf.py                    | 97 +++++++++++++------
 5 files changed, 132 insertions(+), 45 deletions(-)

diff --git a/client/src/components/DataSelection/NetCDFDataConfigurator.vue b/client/src/components/DataSelection/NetCDFDataConfigurator.vue
index b98c36d..81d34d0 100644
--- a/client/src/components/DataSelection/NetCDFDataConfigurator.vue
+++ b/client/src/components/DataSelection/NetCDFDataConfigurator.vue
@@ -194,12 +194,12 @@ export default defineComponent({
       }
       const data = getVariableInformation(newLayerSlice.value);
       if (data.startDate) {
-        sliceLayerRangeStep.value = (data.max / 1e6 - data.min / 1e6) / (data.steps || 1);
-        const startDate = new Date(data.min / 1e6);
-        const endDate = new Date(data.max / 1e6);
+        sliceLayerRangeStep.value = (data.max - data.min) / (data.steps || 1);
+        const startDate = new Date(data.min);
+        const endDate = new Date(data.max);
         const diffMilli = endDate.getTime() - startDate.getTime();
         const differenceInHours = diffMilli / (1000 * 60 * 60);
-        sliceLayerRange.value = [data.min / 1e6, data.max / 1e6];
+        sliceLayerRange.value = [data.min, data.max];
         sliceLayerRangeStep.value = Math.round(differenceInHours / (data.steps || 1)) * (1000 * 60 * 60);
       } else {
         sliceLayerRange.value = [data.min, data.max];
@@ -654,8 +654,8 @@ export default defineComponent({
                   <v-range-slider
                     v-model="sliceLayerRange"
                     :step="sliceLayerRangeStep"
-                    :min="getVariableInformation(newLayerSlice).min / 1e6"
-                    :max="getVariableInformation(newLayerSlice).max / 1e6"
+                    :min="getVariableInformation(newLayerSlice).min"
+                    :max="getVariableInformation(newLayerSlice).max"
                     class="pt-2"
                     hide-details
                   >
@@ -664,10 +664,10 @@ export default defineComponent({
                     </template>
 
                     <template #prepend>
-                      <span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.min / 1e6) }}</span>
+                      <span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.min) }}</span>
                     </template>
                     <template #append>
-                      <span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.max / 1e6) }}</span>
+                      <span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.max) }}</span>
                     </template>
                   </v-range-slider>
                   <v-row align="center" justify="center" class="py-2">
diff --git a/client/src/components/NetCDFLayerConfig.vue b/client/src/components/NetCDFLayerConfig.vue
index 2c5be99..a2c89fd 100644
--- a/client/src/components/NetCDFLayerConfig.vue
+++ b/client/src/components/NetCDFLayerConfig.vue
@@ -38,7 +38,7 @@ export default defineComponent({
             if (found.sliding.variable === 'time') {
               // convert unix timestamp to human readable date YYYY-MM-DD
               try {
-                const date = new Date((mapSlicer[i] as number) / 1e6);
+                const date = new Date((mapSlicer[i] as number) * 1000);
                 // eslint-disable-next-line prefer-destructuring
                 mapSlicer[i] = date.toISOString().split('T')[0];
               } catch (e) {
diff --git a/client/src/utils.ts b/client/src/utils.ts
index 00707e0..5b2f866 100644
--- a/client/src/utils.ts
+++ b/client/src/utils.ts
@@ -174,11 +174,11 @@ const formatNumPrecision = (num: number, range?: number) => {
   return num;
 };
 
-function convertTimestampNSToDatetimeString(timestamp: number): string {
+function convertTimestampNSToDatetimeString(timestamp: number, format = 'date'): string {
   // Convert the nanoseconds to milliseconds
-  const milliseconds = Math.round(timestamp);
+  const seconds = Math.round(timestamp);
   // Create a Date object
-  const date = new Date(milliseconds);
+  const date = new Date(seconds * 1000);
   // Extract the parts of the date in UTC
   const year = date.getUTCFullYear();
   const month = (date.getUTCMonth() + 1).toString().padStart(2, '0'); // Months are 0-indexed
@@ -188,7 +188,57 @@ function convertTimestampNSToDatetimeString(timestamp: number): string {
   const hours = date.getUTCHours().toString().padStart(2, '0');
   const minutes = date.getUTCMinutes().toString().padStart(2, '0');
   // Format the date as YYYYMMDD HH:MM
-  return `${year}${month}${day} ${hours}:${minutes}`;
+  if (format === 'date') {
+    return `${year}-${month}-${day}`;
+  } if (format === 'year') {
+    return `${year}`;
+  }
+  return `${year}-${month}-${day} ${hours}:${minutes}`;
+}
+
+function formatISOToYYMMDD(dateString: string): string | null {
+  const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?$/;
+
+  if (!isoRegex.test(dateString)) {
+    return null; // Not a valid ISO 8601 string
+  }
+
+  try {
+    const date = new Date(dateString);
+    if (Number.isNaN(date.getTime())) {
+      return null; // Invalid date
+    }
+
+    const yy = String(date.getFullYear()).slice(-2);
+    const mm = String(date.getMonth() + 1).padStart(2, '0');
+    const dd = String(date.getDate()).padStart(2, '0');
+
+    return `${yy}-${mm}-${dd}`;
+  } catch {
+    return null;
+  }
+}
+
+function formatCompactToISO(compactNumber: number, format: 'date' | 'datetime' = 'date'): string | null {
+  const compactStr = compactNumber.toString();
+
+  if (!/^(\d{12}|\d{8})$/.test(compactStr)) {
+    return null; // Invalid format
+  }
+
+  const year = compactStr.slice(0, 4);
+  const month = compactStr.slice(4, 6);
+  const day = compactStr.slice(6, 8);
+
+  if (format === 'date') {
+    return `${year}-${month}-${day}`;
+  } if (compactStr.length === 12) {
+    const hour = compactStr.slice(8, 10);
+    const minute = compactStr.slice(10, 12);
+    return `${year}-${month}-${day} ${hour}:${minute}`;
+  }
+
+  return `${year}-${month}-${day}`; // Default fallback
 }
 
 function convert360Longitude(longitude: number): number {
@@ -206,5 +256,7 @@ export {
   formatNumPrecision,
   createColorNumberPairs,
   convertTimestampNSToDatetimeString,
+  formatCompactToISO,
+  formatISOToYYMMDD,
   convert360Longitude,
 };
diff --git a/uvdat/core/rest/netcdf.py b/uvdat/core/rest/netcdf.py
index 267824f..8bfa982 100644
--- a/uvdat/core/rest/netcdf.py
+++ b/uvdat/core/rest/netcdf.py
@@ -88,7 +88,7 @@ def images(self, request, *args, **kwargs):
                     "max": sliding_dim.get('max', step_count),
                     "variable": sliding_dim.get('variable', 'time'),
                 }
-                sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / step_count
+                sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / (step_count - 1)
             response_data = {
                 'netCDFLayer': int(netcdf_layer_id),
                 'parent_bounds': netcdf_layer.bounds.envelope.coords,  # 4-tuple for the parent layer bounds
diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index 8f33586..0c35c27 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -22,6 +22,49 @@
 logger = logging.getLogger(__name__)
 
 
+def convert_time(obj, output='compact'):
+    if isinstance(obj, str):  # Handle ctime (string)
+        dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
+    elif isinstance(obj, np.datetime64):  # Handle datetime64
+        dt_obj = pd.Timestamp(obj).to_pydatetime()
+    elif isinstance(obj, datetime):  # Handle Python datetime objects
+        dt_obj = obj
+    elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+        dt_obj = datetime(obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second)
+    elif isinstance(obj, (int, float)):
+        if obj > 1e10:  # Assume milliseconds timestamp
+            dt_obj = datetime.fromtimestamp(obj / 1000)
+        else:  # Assume seconds timestamp
+            dt_obj = datetime.fromtimestamp(obj)
+    else:
+        return obj  # Return as-is if the type is unrecognized
+
+    if output == 'iso':
+        return dt_obj.isoformat()
+    elif output == 'datetime':
+        return dt_obj
+    elif output == 'compact':
+        return int(dt_obj.strftime('%Y%m%d%H%M%S'))
+    elif output == 'unix':
+        return dt_obj.timestamp()
+
+
+def convert_to_timestamp(obj):
+    if isinstance(obj, str):  # Handle ctime (string)
+        dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
+        return dt_obj.timestamp()
+    elif isinstance(obj, np.datetime64):  # Handle datetime64
+        return (obj - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
+    elif isinstance(obj, datetime):  # Handle Python datetime objects
+        return obj.timestamp()
+    elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+        dt = obj
+        dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+        unix_timestamp = pd.Timestamp(dt_obj).timestamp()
+        return unix_timestamp
+    else:
+        return obj
+
 def create_netcdf_data_layer(file_item, metadata):
     with tempfile.TemporaryDirectory() as temp_dir:
         raw_data_path = Path(temp_dir, 'netcdf.nc')
@@ -51,20 +94,20 @@ def create_netcdf_data_layer(file_item, metadata):
             # Calculate min and max values if the variable has numeric data
             if isinstance(variable.values[0], (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
                 vals = []
-                str_vals = []
                 for item in variable.values:
                     print(item)
                     dt = item
-                    dt_obj = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
-                    unix_timestamp = pd.Timestamp(dt_obj).timestamp()
-                    str_vals.append(pd.Timestamp(dt_obj).isoformat())
-                    vals.append(unix_timestamp)
-                var_min = float(min(vals)) if variable.size > 0 else None
-                var_max = float(max(vals)) if variable.size > 0 else None
-                var_info['min'] = var_min
-                var_info['max'] = var_max
+                    dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
+                    timeobj = convert_time(pd.Timestamp(dt_obj), 'datetime')
+                    vals.append(timeobj)
+                var_min = (min(vals)) if variable.size > 0 else None
+                var_max = (max(vals)) if variable.size > 0 else None
+                var_info['min'] = convert_time(var_min, 'unix')
+                var_info['max'] = convert_time(var_max, 'unix')
+                var_info['startDate'] = convert_time(var_min, 'iso')
+                var_info['endDate'] = convert_time(var_max, 'iso')
+                var_info['timeType'] = 'unix'
                 var_info['steps'] = variable.size
-                # var_info['timeMap'] = str_vals
             else:
                 try:
                     var_min = float(variable.min().values) if variable.size > 0 else None
@@ -72,13 +115,17 @@ def create_netcdf_data_layer(file_item, metadata):
                     if 'datetime' in str(variable.dtype):
                         var_info['startDate'] = str(variable.min().values)
                         var_info['endDate'] = str(variable.max().values)
-                        # str_vals = []
-                        # for item in variable.values:
-                        #     str_vals.append(str(item))
-                        #     var_info['timeMap'] = str_vals
-
-                    var_info['min'] = var_min
-                    var_info['max'] = var_max
+              
+                    if 'time' in var_name:
+                        var_info['min'] = convert_time(var_min, 'unix')
+                        var_info['max'] = convert_time(var_max, 'unix')
+                        var_info['startDate'] = convert_time(var_min, 'iso')
+                        var_info['endDate'] = convert_time(var_max, 'iso')
+
+                        var_info['timeType'] = 'unix'
+                    else:
+                        var_info['min'] = var_min
+                        var_info['max'] = var_max
                     var_info['steps'] = variable.size
 
                     if var_name in description['dimensions'].keys():
@@ -321,18 +368,6 @@ def preview_netcdf_slice(
         return base64_image
 
 
-def convert_to_timestamp(obj):
-    if isinstance(obj, str):  # Handle ctime (string)
-        dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
-        return dt_obj.timestamp()
-    elif isinstance(obj, np.datetime64):  # Handle datetime64
-        return (obj - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
-    elif isinstance(obj, datetime):  # Handle Python datetime objects
-        return obj.timestamp()
-    else:
-        return obj
-
-
 @shared_task
 def create_netcdf_slices(
     netcdf_data_id: int,
@@ -546,8 +581,8 @@ def create_netcdf_slices(
         bounds = None
         if slicer_range:
             slicer_range = [
-                convert_to_timestamp(slicer_range[0]),
-                convert_to_timestamp(slicer_range[1]),
+                convert_time(slicer_range[0], 'unix'),
+                convert_time(slicer_range[1], 'unix'),
             ]
         if variables:
             try:

From 9ac601aaef7923fa0918b4e90a1a1008fbc764d2 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 12 Feb 2025 13:04:40 -0500
Subject: [PATCH 08/16] initial vector feature table rendering

---
 client/src/api/UVDATApi.ts                    |  27 ++
 ...tureChart.vue => PropertyFeatureChart.vue} |   2 +-
 .../FeatureSelection/SelectedFeature.vue      |  37 ++-
 .../FeatureSelection/VectorFeatureChart.vue   | 200 ++++++++++++++
 .../Metadata/FeatureChartEditor.vue           |   6 +-
 client/src/components/PropertiesConfig.vue    |  19 +-
 .../components/TabularData/TableSummary.vue   | 253 ++++++++++++++++++
 client/src/types.ts                           |  66 +++++
 sample_data/fire.json                         |  64 +++++
 uvdat/core/rest/map_layers.py                 |   2 +-
 uvdat/core/rest/netcdf.py                     |  12 +-
 uvdat/core/rest/vector_feature_table_data.py  | 224 +++++++++++++---
 uvdat/core/tasks/netcdf.py                    |  36 ++-
 13 files changed, 887 insertions(+), 61 deletions(-)
 rename client/src/components/FeatureSelection/{RenderFeatureChart.vue => PropertyFeatureChart.vue} (98%)
 create mode 100644 client/src/components/FeatureSelection/VectorFeatureChart.vue
 create mode 100644 client/src/components/TabularData/TableSummary.vue
 create mode 100644 sample_data/fire.json

diff --git a/client/src/api/UVDATApi.ts b/client/src/api/UVDATApi.ts
index 197eeda..e511268 100644
--- a/client/src/api/UVDATApi.ts
+++ b/client/src/api/UVDATApi.ts
@@ -8,6 +8,7 @@ import {
   ContextWithIds,
   Dataset,
   DerivedRegion,
+  FeatureGraphData,
   FileItem,
   LayerCollection,
   LayerCollectionLayer,
@@ -21,6 +22,7 @@ import {
   RasterData,
   RasterMapLayer,
   SimulationType,
+  TableSummary,
   VectorMapLayer,
 } from '../types';
 
@@ -494,4 +496,29 @@ export default class UVdatApi {
   public static async deleteContext(contextId: number): Promise<{ detail: string }> {
     return (await UVdatApi.apiClient.delete(`/contexts/${contextId}/`)).data;
   }
+
+  public static async getVectorTableSummary(layerId: number): Promise<TableSummary> {
+    return (await UVdatApi.apiClient.get('/vectorfeature/tabledata/table-summary/', { params: { layerId } })).data;
+  }
+
+  public static async getFeatureGraphData(
+    tableType: string,
+    vectorFeatureId: number,
+    xAxis: string = 'index',
+    yAxis: string = 'mean_va',
+    filter?: string,
+    filterVal?: string,
+  ): Promise<FeatureGraphData> {
+    const response = await UVdatApi.apiClient.get('/vectorfeature/tabledata/feature-graph/', {
+      params: {
+        tableType,
+        vectorFeatureId,
+        xAxis,
+        yAxis,
+        filter,
+        filterVal,
+      },
+    });
+    return response.data;
+  }
 }
diff --git a/client/src/components/FeatureSelection/RenderFeatureChart.vue b/client/src/components/FeatureSelection/PropertyFeatureChart.vue
similarity index 98%
rename from client/src/components/FeatureSelection/RenderFeatureChart.vue
rename to client/src/components/FeatureSelection/PropertyFeatureChart.vue
index 5fd08b3..80e285e 100644
--- a/client/src/components/FeatureSelection/RenderFeatureChart.vue
+++ b/client/src/components/FeatureSelection/PropertyFeatureChart.vue
@@ -7,7 +7,7 @@ import { drawBarChart } from '../Metadata/drawChart';
 
 // FeatureChart TypeScript interface (as provided)
 export default defineComponent({
-  name: 'RenderFeatureChart',
+  name: 'PropertyFeatureChart',
   props: {
     featureChart: {
       type: Object as PropType<FeatureChartWithData>,
diff --git a/client/src/components/FeatureSelection/SelectedFeature.vue b/client/src/components/FeatureSelection/SelectedFeature.vue
index 1838195..6b30e08 100644
--- a/client/src/components/FeatureSelection/SelectedFeature.vue
+++ b/client/src/components/FeatureSelection/SelectedFeature.vue
@@ -3,13 +3,15 @@ import {
   PropType, Ref, computed, defineComponent, onMounted, ref,
 } from 'vue';
 import MapStore from '../../MapStore';
-import { ClickedProps, FeatureChartWithData } from '../../types';
+import { ClickedProps, FeatureChartWithData, VectorFeatureTableGraph } from '../../types';
 import { colorGenerator } from '../../map/mapColors';
-import RenderFeatureChart from './RenderFeatureChart.vue';
+import PropertyFeatureChart from './PropertyFeatureChart.vue';
+import VectorFeatureChart from './VectorFeatureChart.vue';
 
 export default defineComponent({
   components: {
-    RenderFeatureChart,
+    PropertyFeatureChart,
+    VectorFeatureChart,
   },
   props: {
     data: {
@@ -28,6 +30,8 @@ export default defineComponent({
     const mapLayerId = ref(props.data.layerId);
     const featureId = ref(props.data.id as number);
     const enabledChartPanels: Ref<number[]> = ref([]);
+    const enabledVectorChartPanel: Ref<number[]> = ref([]);
+    const vectorGraphs: Ref<VectorFeatureTableGraph[]> = ref([]);
     const processLayerProps = () => {
       const found = MapStore.selectedVectorMapLayers.value.find((item) => item.id === props.data.layerId);
       if (found?.default_style.properties) {
@@ -65,6 +69,9 @@ export default defineComponent({
           });
         }
       }
+      if (found?.default_style.vectorFeatureTableGraphs) {
+        vectorGraphs.value = found.default_style.vectorFeatureTableGraphs;
+      }
     };
     onMounted(() => processLayerProps());
 
@@ -81,8 +88,10 @@ export default defineComponent({
       featureId,
       featureCharts,
       enabledChartPanels,
+      enabledVectorChartPanel,
       selectedFeatureExpanded,
       toggleFeatureExpaned,
+      vectorGraphs,
     };
   },
 });
@@ -137,7 +146,24 @@ export default defineComponent({
             </v-icon> {{ featureChart.name }}
           </v-expansion-panel-title>
           <v-expansion-panel-text>
-            <render-feature-chart :feature-chart="featureChart" />
+            <property-feature-chart :feature-chart="featureChart" />
+          </v-expansion-panel-text>
+        </v-expansion-panel>
+      </v-expansion-panels>
+      <v-expansion-panels v-if="vectorGraphs.length" v-model="enabledVectorChartPanel" multiple>
+        <v-expansion-panel
+          v-for="vectorGraph, index in vectorGraphs"
+          :key="`${vectorGraph.name}_${index}`"
+          :value="index"
+          class="ma-0 pa-0"
+        >
+          <v-expansion-panel-title>
+            <v-icon class="pr-2">
+              mdi-chart-line
+            </v-icon> {{ vectorGraph.name }}
+          </v-expansion-panel-title>
+          <v-expansion-panel-text class="ma-0">
+            <vector-feature-chart :graph-info="vectorGraph" :vector-feature-id="featureId" />
           </v-expansion-panel-text>
         </v-expansion-panel>
       </v-expansion-panels>
@@ -146,4 +172,7 @@ export default defineComponent({
 </template>
 
 <style scoped>
+.v-expansion-panel-text>>> .v-expansion-panel-text__wrapper {
+  padding: 8px !important;
+}
 </style>
diff --git a/client/src/components/FeatureSelection/VectorFeatureChart.vue b/client/src/components/FeatureSelection/VectorFeatureChart.vue
new file mode 100644
index 0000000..a33f7ab
--- /dev/null
+++ b/client/src/components/FeatureSelection/VectorFeatureChart.vue
@@ -0,0 +1,200 @@
+<script lang="ts">
+import {
+  PropType, defineComponent, nextTick, ref, watch,
+} from 'vue';
+import * as d3 from 'd3';
+import UVdatApi from '../../api/UVDATApi';
+import { FeatureGraphData, VectorFeatureTableGraph } from '../../types';
+
+type ParameterGraph = Record<string, {
+  data: [number, number][];
+  filterVal: string;
+}>;
+
+export default defineComponent({
+  name: 'FeatureGraph',
+  props: {
+    graphInfo: {
+      type: Object as PropType<VectorFeatureTableGraph>,
+      required: true,
+    },
+    vectorFeatureId: {
+      type: Number as PropType<number>,
+      required: true,
+    },
+  },
+  setup(props) {
+    const graphContainer = ref<SVGSVGElement | null>(null);
+    const graphDialogContainer = ref<SVGSVGElement | null>(null);
+    const graphData = ref<FeatureGraphData | null>(null);
+    const dialogVisible = ref(false);
+
+    // Fetch feature graph data when component is mounted or props change
+    const fetchFeatureGraphData = async () => {
+      try {
+        const data = await UVdatApi.getFeatureGraphData(
+          props.graphInfo.type, // Use graphInfo.type (tableType) instead of mapLayerId
+          props.vectorFeatureId,
+          props.graphInfo.xAxis,
+          props.graphInfo.yAxis,
+          props.graphInfo.filterColumn,
+          props.graphInfo.filterValue,
+        );
+        graphData.value = data;
+        renderGraph(data);
+      } catch (error) {
+        // eslint-disable-next-line no-console
+        console.error('Error fetching feature graph data:', error);
+      }
+    };
+
+    // Render graph using D3
+    const renderGraph = (data: FeatureGraphData, container = 'graphContainer') => {
+      const localContainer = container === 'graphContainer' ? graphContainer : graphDialogContainer;
+      if (!localContainer.value || !data) return;
+
+      const svg = d3.select(localContainer.value);
+      svg.selectAll('*').remove(); // Clear any existing content in the SVG
+
+      const margin = {
+        top: 20, right: container === 'graphContainer' ? 0 : 20, bottom: 40, left: 40,
+      };
+
+      // Set the maximum width to 250px
+      const width = localContainer.value?.clientWidth || 250 - margin.left - margin.right;
+      const height = 400 - margin.top - margin.bottom;
+
+      const x = d3.scaleLinear().range([0, width]);
+      const y = d3.scaleLinear().range([height, 0]);
+
+      const line = d3.line()
+        .x((d: [number, number]) => x(d[0]))
+        .y((d: [number, number]) => y(d[1]));
+
+      let dataForGraph: { data: [number, number][], filterVal?: string } | undefined;
+
+      // Check for default data or apply filter if necessary
+      if (data.graphs.default) {
+        dataForGraph = data.graphs.default; // Use default data if no filter
+      } else if (props.graphInfo.filterValue && props.graphInfo.filterColumn) {
+        const filteredData = data.graphs as ParameterGraph;
+        dataForGraph = filteredData[props.graphInfo.filterValue]; // Apply filter by value
+      }
+
+      if (!dataForGraph) {
+        return; // Return if no data is available
+      }
+
+      // Set the domain for the axes, ensuring we handle empty arrays correctly
+      const xExtent = d3.extent(dataForGraph.data.map((item) => item[0]));
+      const yExtent = d3.extent(dataForGraph.data.map((item) => item[1]));
+
+      // Fallback to zero if data is empty
+      x.domain(xExtent[0] !== undefined ? xExtent : [0, 1]);
+      y.domain(yExtent[0] !== undefined ? yExtent : [0, 1]);
+
+      // Create the graph container
+      const g = svg.append('g')
+        .attr('transform', `translate(${margin.left},${margin.top})`);
+
+      // Add the line path to the graph
+      svg.append('path')
+        .attr('fill', 'none')
+        .attr('stroke', 'steelblue')
+        .attr('stroke-width', 1.5)
+        .attr('d', line(dataForGraph.data.sort((a, b) => a[0] - b[0])))
+        .attr('transform', `translate(${margin.left},${margin.top})`);
+
+      // Add the X-axis
+      g.append('g')
+        .attr('class', 'x axis')
+        .attr('transform', `translate(0,${height})`)
+        .call(d3.axisBottom(x).ticks(5));
+
+      // Add the Y-axis
+      g.append('g')
+        .attr('class', 'y axis')
+        .call(d3.axisLeft(y));
+    };
+
+    // Watch for prop changes and refetch data
+    watch(
+      () => [props.graphInfo, props.vectorFeatureId],
+      () => fetchFeatureGraphData(),
+      { immediate: true },
+    );
+
+    // Open the dialog to display a larger graph
+    const openDialog = () => {
+      dialogVisible.value = true;
+      nextTick(() => {
+        if (graphData.value) {
+          renderGraph(graphData.value, 'graphDialogContainer');
+        }
+      });
+    };
+
+    return {
+      graphContainer,
+      graphDialogContainer,
+      graphData,
+      dialogVisible,
+      openDialog,
+    };
+  },
+});
+</script>
+
+<template>
+  <div>
+    <!-- Button to open dialog -->
+    <v-btn color="primary" @click="openDialog">
+      View Larger Graph
+    </v-btn>
+
+    <!-- Graph container -->
+    <svg ref="graphContainer" width="100%" height="400" class="selectedFeatureSVG" />
+
+    <!-- Dialog for larger chart -->
+    <v-dialog v-model="dialogVisible" max-width="800px">
+      <v-card>
+        <v-card-title>
+          <span class="headline">Feature Graph</span>
+        </v-card-title>
+        <v-card-text>
+          <svg ref="graphDialogContainer" width="100%" height="500" />
+        </v-card-text>
+        <v-card-actions>
+          <v-btn @click="dialogVisible = false">
+            Close
+          </v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
+  </div>
+</template>
+
+<style scoped>
+  .selectedFeatureSVG {
+    max-width: 250px;
+    width: 100%;
+    height: auto;
+  }
+
+  .line {
+    fill: none;
+    stroke: steelblue;
+    stroke-width: 1.5;
+  }
+
+  .axis path,
+  .axis line {
+    fill: none;
+    shape-rendering: crispEdges;
+  }
+
+  .x.axis text,
+  .y.axis text {
+    font-size: 12px;
+  }
+</style>
diff --git a/client/src/components/Metadata/FeatureChartEditor.vue b/client/src/components/Metadata/FeatureChartEditor.vue
index 03045ac..b160a4f 100644
--- a/client/src/components/Metadata/FeatureChartEditor.vue
+++ b/client/src/components/Metadata/FeatureChartEditor.vue
@@ -7,13 +7,13 @@ import { FeatureChart } from '../../types';
 import MapStore from '../../MapStore';
 import { drawBarChart } from './drawChart'; // Separate drawChart function
 import { colorGenerator } from '../../map/mapColors';
-import RenderFeatureChart from '../FeatureSelection/RenderFeatureChart.vue';
+import PropertyFeatureChart from '../FeatureSelection/PropertyFeatureChart.vue';
 
 export default defineComponent({
   name: 'FeatureChartEditor',
   components: {
     draggable,
-    RenderFeatureChart,
+    PropertyFeatureChart,
   },
   props: {
     layerId: {
@@ -278,7 +278,7 @@ export default defineComponent({
           </v-icon> {{ chart.name }}
         </v-expansion-panel-title>
         <v-expansion-panel-text>
-          <render-feature-chart :feature-chart="{ ...chart, data: generatedData }" :max-width="500" />
+          <property-feature-chart :feature-chart="{ ...chart, data: generatedData }" :max-width="500" />
         </v-expansion-panel-text>
       </v-expansion-panel>
     </v-expansion-panels>
diff --git a/client/src/components/PropertiesConfig.vue b/client/src/components/PropertiesConfig.vue
index 9bf2540..149707b 100644
--- a/client/src/components/PropertiesConfig.vue
+++ b/client/src/components/PropertiesConfig.vue
@@ -5,12 +5,14 @@ import {
 import AvailableProperties from './Metadata/AvailableProperties.vue';
 import MetadataSettings from './Metadata/MetadataSettings.vue';
 import SelectedFeatureChartCard from './Metadata/SelectedFeatureChartCard.vue';
+import TableSummary from './TabularData/TableSummary.vue';
 
 export default defineComponent({
   components: {
     AvailableProperties,
     MetadataSettings,
     SelectedFeatureChartCard,
+    TableSummary,
   },
   props: {
     layerId: {
@@ -19,7 +21,7 @@ export default defineComponent({
     },
   },
   setup() {
-    const tab: Ref<'availableProperties' | 'settings' | 'charts'> = ref('availableProperties');
+    const tab: Ref<'availableProperties' | 'settings' | 'charts' | 'tabularData'> = ref('availableProperties');
 
     return {
       tab,
@@ -72,12 +74,27 @@ export default defineComponent({
         </v-icon>
       </template>
     </v-tooltip>
+    <v-tooltip text="Tabular Data">
+      <template #activator="{ props }">
+        <v-icon
+          class="icon-center"
+          :class="{ 'selected-tab': tab === 'tabularData' }"
+          v-bind="props"
+          color="primary"
+          size="x-small"
+          @click="tab = 'tabularData'"
+        >
+          mdi-table
+        </v-icon>
+      </template>
+    </v-tooltip>
 
     <v-spacer />
   </v-row>
   <AvailableProperties v-if="tab === 'availableProperties'" :layer-id="layerId" />
   <MetadataSettings v-else-if="tab === 'settings'" :layer-id="layerId" />
   <SelectedFeatureChartCard v-else-if="tab === 'charts'" :layer-id="layerId" />
+  <TableSummary v-else-if="tab === 'tabularData'" :layer-id="layerId" />
 </template>
 
 <style scoped>
diff --git a/client/src/components/TabularData/TableSummary.vue b/client/src/components/TabularData/TableSummary.vue
new file mode 100644
index 0000000..4a66b47
--- /dev/null
+++ b/client/src/components/TabularData/TableSummary.vue
@@ -0,0 +1,253 @@
+<script lang="ts">
+import {
+  Ref, defineComponent, onMounted, ref,
+} from 'vue';
+import { TableSummary, VectorFeatureTableGraph } from '../../types';
+import UVdatApi from '../../api/UVDATApi';
+import MapStore from '../../MapStore';
+
+export default defineComponent({
+  name: 'VectorTableSummary',
+  props: {
+    layerId: {
+      type: Number,
+      required: true,
+    },
+  },
+  setup(props) {
+    const summaryData = ref<TableSummary | null>(null);
+    const loading = ref(true);
+    const error = ref<string | null>(null);
+    const tableChartDialog = ref(false);
+    const activeTab: Ref<'summary' | 'graphs'> = ref('summary'); // Track active tab (0 for summary, 1 for graphing)
+    const selectedTableName = ref<string>('Table Name');
+    const selectedTableType = ref<string | null>(null);
+    const selectedXColumn = ref<string | null>(null);
+    const selectedYColumn = ref<string | null>(null);
+    const selectedFilterColumn = ref<string>('');
+    const filterValue = ref<string>('');
+    const graphs = ref<VectorFeatureTableGraph[]>([]);
+
+    const getStyleVectorFeatureGraphs = () => {
+      const found = MapStore.selectedVectorMapLayers.value.find((item) => item.id === props.layerId);
+      if (found && found.default_style.vectorFeatureTableGraphs) {
+        graphs.value = found.default_style.vectorFeatureTableGraphs;
+      }
+    };
+
+    const updateStyleGraphs = (updatedGraphs: VectorFeatureTableGraph[]) => {
+      const found = MapStore.selectedVectorMapLayers.value.find((item) => item.id === props.layerId);
+      if (found && found.default_style) {
+        found.default_style.vectorFeatureTableGraphs = updatedGraphs;
+      }
+    };
+
+    const editingGraphIndex = ref<number | null>(null); // Track the index of the graph being edited
+
+    const vectorTableSummary = async (layerId: number) => {
+      try {
+        loading.value = true;
+        summaryData.value = await UVdatApi.getVectorTableSummary(layerId);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } catch (err: any) {
+        error.value = err.response?.data?.error || 'Failed to fetch data';
+      } finally {
+        loading.value = false;
+      }
+    };
+
+    const resetGraphForm = () => {
+      selectedXColumn.value = null;
+      selectedYColumn.value = null;
+      selectedFilterColumn.value = '';
+      filterValue.value = '';
+    };
+
+    const addSaveGraph = () => {
+      if (selectedTableType.value && selectedXColumn.value && selectedYColumn.value) {
+        const graph: VectorFeatureTableGraph = {
+          name: selectedTableName.value,
+          type: selectedTableType.value,
+          xAxis: selectedXColumn.value,
+          yAxis: selectedYColumn.value,
+          filterColumn: selectedFilterColumn.value ? selectedFilterColumn.value : undefined,
+          filterValue: filterValue.value ? filterValue.value : undefined,
+        };
+
+        if (editingGraphIndex.value !== null) {
+          // If editing, update the existing graph
+          graphs.value[editingGraphIndex.value] = graph;
+          editingGraphIndex.value = null;
+        } else {
+          // Otherwise, add a new graph
+          graphs.value.push(graph);
+        }
+
+        updateStyleGraphs(graphs.value);
+        // Clear the form after adding/updating the graph
+        resetGraphForm();
+      }
+    };
+
+    const editGraph = (index: number) => {
+      const graph = graphs.value[index];
+      // eslint-disable-next-line prefer-destructuring
+      selectedTableType.value = graph.type; // Assuming the table name is part of the graph name
+      selectedTableName.value = graph.name;
+      selectedXColumn.value = graph.xAxis;
+      selectedYColumn.value = graph.yAxis;
+      selectedFilterColumn.value = graph.filterColumn || '';
+      filterValue.value = graph.filterValue || ' ';
+      editingGraphIndex.value = index;
+    };
+
+    const deleteGraph = (index: number) => {
+      graphs.value.splice(index, 1);
+    };
+
+    onMounted(() => {
+      vectorTableSummary(props.layerId);
+      getStyleVectorFeatureGraphs();
+    });
+
+    return {
+      summaryData,
+      loading,
+      error,
+      tableChartDialog,
+      activeTab,
+      selectedTableName,
+      selectedTableType,
+      selectedXColumn,
+      selectedYColumn,
+      selectedFilterColumn,
+      editingGraphIndex,
+      filterValue,
+      graphs,
+      addSaveGraph,
+      editGraph,
+      deleteGraph,
+    };
+  },
+});
+</script>
+
+<template>
+  <v-row dense align="center" justify="center">
+    <h3>Tabular Data/Graphs</h3>
+    <v-spacer />
+    <v-icon @click="tableChartDialog = true">
+      mdi-cog
+    </v-icon>
+  </v-row>
+  <v-row v-for="(graph, index) in graphs" :key="index">
+    <v-col>
+      {{ graph.name }}
+    </v-col>
+    <v-col>
+      {{ `${graph.xAxis} vs ${graph.yAxis}` }}
+    </v-col>
+  </v-row>
+
+  <v-dialog v-model="tableChartDialog" max-width="800px">
+    <v-card>
+      <v-card-title>Vector Table Summary</v-card-title>
+      <v-tabs v-model="activeTab">
+        <v-tab value="summary">
+          Summary
+        </v-tab>
+        <v-tab value="graphs">
+          Graphs
+        </v-tab>
+      </v-tabs>
+      <v-card-text v-if="activeTab === 'summary'">
+        <div v-if="loading">
+          Loading...
+        </div>
+        <div v-else-if="error" class="error-text">
+          {{ error }}
+        </div>
+        <div v-else-if="summaryData">
+          <p><strong>Vector Feature Count:</strong> {{ summaryData.vectorFeatureCount }}</p>
+          <v-expansion-panels>
+            <v-expansion-panel v-for="(table, type) in summaryData.tables" :key="type">
+              <v-expansion-panel-title>
+                {{ type }} ({{ table.tableCount }} tables)
+              </v-expansion-panel-title>
+              <v-expansion-panel-text>
+                <v-expansion-panels>
+                  <v-expansion-panel v-for="column in table.columns" :key="column">
+                    <v-expansion-panel-title>{{ column }}</v-expansion-panel-title>
+                    <v-expansion-panel-text>
+                      <p>{{ table.summary[column] }}</p>
+                    </v-expansion-panel-text>
+                  </v-expansion-panel>
+                </v-expansion-panels>
+              </v-expansion-panel-text>
+            </v-expansion-panel>
+          </v-expansion-panels>
+        </div>
+      </v-card-text>
+
+      <v-card-text v-if="activeTab === 'graphs'">
+        <v-card-text>
+          <v-form>
+            <v-text-field
+              v-model="selectedTableName"
+              label="Table Name"
+            />
+            <v-select
+              v-model="selectedTableType"
+              :items="Object.keys(summaryData?.tables || {})"
+              label="Select Table"
+            />
+            <v-select
+              v-model="selectedXColumn"
+              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
+              label="Select X Axis"
+            />
+            <v-select
+              v-model="selectedYColumn"
+              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
+              label="Select Y Axis"
+            />
+            <v-select
+              v-model="selectedFilterColumn"
+              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
+              label="Select Filter Column"
+            />
+            <v-text-field
+              v-model="filterValue"
+              label="Filter Value"
+              :disabled="!selectedFilterColumn"
+            />
+            <v-btn :disabled="!selectedTableType || !selectedXColumn || !selectedYColumn" color="primary" @click="addSaveGraph">
+              {{ editingGraphIndex !== null ? 'Update' : 'Add' }} Graph
+            </v-btn>
+          </v-form>
+          <v-divider />
+          <v-list>
+            <v-list-item v-for="(graph, index) in graphs" :key="index">
+              <v-list-item-title>{{ graph.name }}</v-list-item-title>
+              <v-list-item-subtitle>
+                <div>{{ graph.type }}</div>
+                <div>{{ graph.xAxis }} vs {{ graph.yAxis }} {{ graph.filterValue ? `(Filter - ${graph.filterValue}` : '' }})</div>
+              </v-list-item-subtitle>
+              <v-list-item-subtitle v-if="graph.filterColumn">
+                Filter: {{ graph.filterColumn }} = {{ graph.filterValue }}
+              </v-list-item-subtitle>
+              <v-list-item-action>
+                <v-icon color="warning" @click="editGraph(index)">
+                  mdi-pencil
+                </v-icon>
+                <v-icon color="error" @click="deleteGraph(index)">
+                  mdi-delete
+                </v-icon>
+              </v-list-item-action>
+            </v-list-item>
+          </v-list>
+        </v-card-text>
+      </v-card-text>
+    </v-card>
+  </v-dialog>
+</template>
diff --git a/client/src/types.ts b/client/src/types.ts
index 63d8dda..20587c4 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -394,6 +394,7 @@ export interface VectorMapLayer extends AbstractMapLayer {
     filters?: Filter[];
     charts?: CustomChart[];
     selectedFeatureCharts?: FeatureChart[];
+    vectorFeatureTableGraphs?: VectorFeatureTableGraph[];
 
   }
 }
@@ -734,3 +735,68 @@ export interface VectorTimeSeries {
 }
 
 export type FeatureChartWithData = FeatureChart & { data: { key: string, value: number; color: string }[] };
+
+export interface TableSummary {
+  vectorFeatureCount: number;
+  tables: Record<string, TableInfo>;
+}
+
+interface TableInfo {
+  tableCount: number;
+  columns: string[];
+  summary: Record<string, ColumnSummary>;
+}
+
+type ColumnSummary =
+  | NumberColumnSummary
+  | StringColumnSummary
+  | ParameterColumnSummary;
+
+interface NumberColumnSummary {
+  type: 'number';
+  min: number;
+  max: number;
+  value_count: number;
+}
+
+interface StringColumnSummary {
+  type: 'string';
+  values: string[];
+  value_count: number;
+}
+
+interface ParameterColumnSummary {
+  type: 'parameter_cd';
+  parameters: Record<string, ParameterStats>;
+}
+
+interface ParameterStats {
+  parameter_cd: string;
+  parameter_name: string;
+  min: number;
+  max: number;
+  mean: number;
+}
+
+export interface VectorFeatureTableGraph {
+  name: string;
+  type: string;
+  xAxis: string;
+  yAxis: string;
+  filterColumn?: string;
+  filterValue?: string;
+}
+
+export interface FeatureGraphData {
+  table_name: string;
+  graphs:
+  Record<string, {
+    data:[number, number][];
+    filterVal: string;
+  }>
+  | {
+    default: {
+      data: [number, number][];
+    };
+  };
+}
diff --git a/sample_data/fire.json b/sample_data/fire.json
new file mode 100644
index 0000000..583d456
--- /dev/null
+++ b/sample_data/fire.json
@@ -0,0 +1,64 @@
+[
+    {
+        "type": "Context",
+        "name": "WildFire Data",
+        "default_map_center": [
+            34.8019,
+            -86.1794
+        ],
+        "default_map_zoom": 6,
+        "datasets": [
+            {
+                "name": "WildFire Data",
+                "description": "MODIS 1km USA (Conterminous) + Hawaii 48h shapefile. From: https://firms.modaps.eosdis.nasa.gov/active_fire/",
+                "category": "region",
+                "metadata": {},
+                "files": [
+                    {
+                        "path": "./data/la-fire-2025/modis-1-48.zip",
+                        "url": "https://data.kitware.com/api/v1/item/67aa368b091a514e82eec908/download",
+                        "name": "MODIS 1km 48h USA + Hawaii",
+                        "type": "zip",
+                        "metadata": {}
+                    },
+                    {
+
+                        "path": "./data/la-fire-2025/current_fire_perimeters.geojson",
+                        "url": "https://data.kitware.com/api/v1/item/67aa368c091a514e82eec90b/download",
+                        "name": "USA Palisades + Eaton Wildfire Perimeters",
+                        "type": "geojson",
+                        "metadata": {}
+                    },
+                    {
+                        "path": "./data/la-fire-2025/current_fire_incidents.geojson",
+                        "url": "https://data.kitware.com/api/v1/item/67aa368c091a514e82eec90e/download",
+                        "name": "USA Palisades + Eaton Wildfire Incidents",
+                        "type": "geojson",
+                        "metadata": {}
+                    },
+                    {
+                        "path": "./data/la-fire-2025/ca_perimeters_nifc_firis.geojson",
+                        "url": "https://data.kitware.com/api/v1/item/67aa33ed091a514e82eec8ff/download",
+                        "name": "California Fire Perimeters",
+                        "type": "geojson",
+                        "metadata": {}
+                    },
+                    {
+                        "path": "./data/la-fire-2025/palidades-2025-01-14.tif",
+                        "url": "https://data.kitware.com/api/v1/item/67aa368a091a514e82eec905/download",
+                        "name": "Palisades Satellite Imagery",
+                        "type": "tif",
+                        "metadata": {}
+                    },
+                    {
+                        "path": "./data/la-fire-2025/eaton-2025-01-14_to_19.tif",
+                        "url": "https://data.kitware.com/api/v1/item/67aa353a091a514e82eec902/download",
+                        "name": "Eaton/Altadena Satellite Imagery",
+                        "type": "tif",
+                        "metadata": {}
+                    }
+                ]
+            }
+        ]
+    }
+]
\ No newline at end of file
diff --git a/uvdat/core/rest/map_layers.py b/uvdat/core/rest/map_layers.py
index 15f78a0..f66cc2f 100644
--- a/uvdat/core/rest/map_layers.py
+++ b/uvdat/core/rest/map_layers.py
@@ -13,8 +13,8 @@
 
 from uvdat.core.models import (
     LayerRepresentation,
-    NetCDFLayer,
     NetCDFData,
+    NetCDFLayer,
     RasterMapLayer,
     VectorFeature,
     VectorMapLayer,
diff --git a/uvdat/core/rest/netcdf.py b/uvdat/core/rest/netcdf.py
index 8bfa982..095cb0d 100644
--- a/uvdat/core/rest/netcdf.py
+++ b/uvdat/core/rest/netcdf.py
@@ -84,15 +84,17 @@ def images(self, request, *args, **kwargs):
             step_count = netcdf_layer.parameters.get('stepCount', None)
             if sliding_dim and step_count:
                 sliding_data = {
-                    "min": sliding_dim.get('min', 0),
-                    "max": sliding_dim.get('max', step_count),
-                    "variable": sliding_dim.get('variable', 'time'),
+                    'min': sliding_dim.get('min', 0),
+                    'max': sliding_dim.get('max', step_count),
+                    'variable': sliding_dim.get('variable', 'time'),
                 }
-                sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / (step_count - 1)
+                sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / (
+                    step_count - 1
+                )
             response_data = {
                 'netCDFLayer': int(netcdf_layer_id),
                 'parent_bounds': netcdf_layer.bounds.envelope.coords,  # 4-tuple for the parent layer bounds
-                "sliding": sliding_data,
+                'sliding': sliding_data,
                 'images': [image.image.url for image in netcdf_images],  # Only return the image URL
             }
 
diff --git a/uvdat/core/rest/vector_feature_table_data.py b/uvdat/core/rest/vector_feature_table_data.py
index e653274..5bba118 100644
--- a/uvdat/core/rest/vector_feature_table_data.py
+++ b/uvdat/core/rest/vector_feature_table_data.py
@@ -1,16 +1,18 @@
 from collections import defaultdict
+import logging
 
+from django.contrib.gis.geos import Polygon
 from django.db import connection
 from django.http import HttpResponse
 from rest_framework import mixins, status, viewsets
-from rest_framework.decorators import action, permission_classes
+from rest_framework.decorators import action
 from rest_framework.exceptions import ValidationError
-from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 
 from uvdat.core.models import VectorFeature, VectorFeatureRowData, VectorFeatureTableData
 from uvdat.core.rest.serializers import VectorFeatureTableDataSerializer
 
+logger = logging.getLogger(__name__)
 # TODO THIS IS INCOMPLETE
 VECTOR_TILE_DYNAMIC_SQL = """
     WITH tile_bounds AS (
@@ -82,27 +84,92 @@ def table_summary(self, request, *args, **kwargs):
 
         tables = VectorFeatureTableData.objects.filter(map_layer=layer_id)
         feature_count = VectorFeature.objects.filter(map_layer=layer_id).count()
-        # Create a defaultdict to store types and their corresponding columns
+
         type_columns_map = defaultdict(set)
         table_count_map = defaultdict(int)
+        column_summaries = defaultdict(lambda: defaultdict(dict))
 
-        # Iterate over the tables and collect the types and columns
+        # Iterate over tables to collect type-based column definitions and summary stats
         for table in tables:
-            if len(type_columns_map[table.type]) == 0:
-                type_columns_map[table.type] = set(table.columns)
-            elif type_columns_map[table.type] != set(table.columns):
+            table_type = table.type
+            table_summary = table.summary or {}
+
+            # Ensure all tables of the same type have consistent columns
+            if len(type_columns_map[table_type]) == 0:
+                type_columns_map[table_type] = set(table.columns)
+            elif type_columns_map[table_type] != set(table.columns):
                 raise ValidationError(
-                    f"Table Columns don't match for all tables: {type_columns_map[table.type]} vs { set(table.columns)}"
+                    f'Table Columns do not match for all tables: {type_columns_map[table_type]} vs {set(table.columns)}'
                 )
-            table_count_map[table.type] += 1
 
-        # Convert the defaultdict to a normal dict (optional)
-        type_columns_map = dict(type_columns_map)
+            table_count_map[table_type] += 1
+
+            # Aggregate summary statistics per column
+            for column, stats in table_summary.items():
+                if 'type' not in column_summaries[table_type][column]:
+                    column_summaries[table_type][column]['type'] = stats.get('type')
+
+                if stats.get('type') == 'number':
+                    column_summaries[table_type][column]['min'] = min(
+                        column_summaries[table_type][column].get('min', float('inf')),
+                        stats.get('min', float('inf')),
+                    )
+                    column_summaries[table_type][column]['max'] = max(
+                        column_summaries[table_type][column].get('max', float('-inf')),
+                        stats.get('max', float('-inf')),
+                    )
+                    column_summaries[table_type][column]['value_count'] = column_summaries[
+                        table_type
+                    ][column].get('value_count', 0) + stats.get('value_count', 0)
+
+                elif stats.get('type') == 'string':
+                    existing_values = set(column_summaries[table_type][column].get('values', []))
+                    new_values = set(stats.get('values', []))
+                    column_summaries[table_type][column]['values'] = list(
+                        existing_values.union(new_values)
+                    )
+                    column_summaries[table_type][column]['value_count'] = column_summaries[
+                        table_type
+                    ][column].get('value_count', 0) + stats.get('value_count', 0)
+
+                # Special USGS Parameter Type
+                elif stats.get('type') == 'parameter_cd':
+                    if 'parameters' not in column_summaries[table_type][column]:
+                        column_summaries[table_type][column]['parameters'] = {}
+
+                    for param_cd, param_stats in stats.items():
+                        if param_cd == 'type':
+                            continue
+                        if param_cd not in column_summaries[table_type][column]['parameters']:
+                            column_summaries[table_type][column]['parameters'][
+                                param_cd
+                            ] = param_stats
+                        else:
+                            column_summaries[table_type][column]['parameters'][param_cd]['min'] = (
+                                min(
+                                    column_summaries[table_type][column]['parameters'][param_cd][
+                                        'min'
+                                    ],
+                                    param_stats['min'],
+                                )
+                            )
+                            column_summaries[table_type][column]['parameters'][param_cd]['max'] = (
+                                max(
+                                    column_summaries[table_type][column]['parameters'][param_cd][
+                                        'max'
+                                    ],
+                                    param_stats['max'],
+                                )
+                            )
+
+        # Construct the response
         output = {'vectorFeatureCount': feature_count, 'tables': {}}
-        for key in type_columns_map:
-            output['tables'][key] = {
-                'tableCount': table_count_map[key],
-                'columns': type_columns_map[key],
+        for table_type in type_columns_map:
+            output['tables'][table_type] = {
+                'type': table_type,
+                'tableCount': table_count_map[table_type],
+                'columns': list(type_columns_map[table_type]),
+                'summary': column_summaries[table_type],
             }
 
         return Response(output, status=status.HTTP_200_OK)
@@ -114,10 +181,8 @@ def feature_graph(self, request, *args, **kwargs):
         x_axis = request.query_params.get('xAxis', 'index')
         y_axis = request.query_params.get('yAxis', 'mean_va')
         filter_param = request.query_params.get('filter', 'parameter_cd')
-        filter_vals = request.query_params.getlist(
-            'filterVals', ['00060', '00065']
-        )  # ['00060', '00065']
-
+        filter_val = request.query_params.get('filterVal', '00065')
+        logger.info(f'filter_vals: {filter_val}')
         if not table_type:
             return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
 
@@ -125,42 +190,123 @@ def feature_graph(self, request, *args, **kwargs):
             table = VectorFeatureTableData.objects.get(
                 type=table_type, vector_feature=vector_feature
             )
-            table_data = {'table_name': table.name, 'tables': []}
+            table_data = {'table_name': table.name}
             x_axis_index = table.columns.index(x_axis)
             y_axis_index = table.columns.index(y_axis)
-            if filter_param and len(filter_vals):
+            if filter_param and filter_val:
                 filter_param_index = table.columns.index(filter_param)
-            if filter_param and len(filter_vals):
-                table_data['data'] = {}
-                for filter_val in filter_vals:
-                    table_data['data'][filter_val] = {
-                        'x_data': [],
-                        'y_data': [],
-                        'filterVal': filter_val,
-                    }
+            if filter_param and filter_val:
+                table_data['graphs'] = {}
+                table_data['graphs'][filter_val] = {
+                    'data': [],
+                    'filterVal': filter_val,
+                }
             else:
-                table_data['data'] = {'default': {'x_data': [], 'y_data': []}}
+                table_data['graphs'] = {'default': {'data': []}}
             rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
             for row in rows:
                 row_data = row.row_data
                 filter_row = row_data[filter_param_index]
                 x_val = row_data[x_axis_index]
                 y_val = row_data[y_axis_index]
-                if filter_param and len(filter_vals):
-                    for filter_val in filter_vals:
-                        if filter_row == filter_val:
-                            table_data['data'][filter_val]['x_data'].append(x_val)
-                            table_data['data'][filter_val]['y_data'].append(y_val)
+                if filter_param and filter_val:
+                    if filter_row == filter_val:
+                        table_data['graphs'][filter_val]['data'].append([x_val, y_val])
                 else:
-                    table_data['data']['default']['x_data'].append(x_val)
-                    table_data['data']['default']['y_data'].append(y_val)
-            print(table_data)
+                    table_data['graphs']['default']['data'].append([x_val, y_val])
             return Response(table_data, status=status.HTTP_200_OK)
         except VectorFeatureTableData.DoesNotExist:
             return Response({'error': 'Table not found'}, status=status.HTTP_404_NOT_FOUND)
 
+    @action(detail=False, methods=['get'], url_path='map-layer-feature-graph')
+    def map_layer_feature_graph(self, request, *args, **kwargs):
+        table_type = request.query_params.get('tableType')  # Required
+        map_layer_id = request.query_params.get('mapLayerId')  # Required
+        x_axis = request.query_params.get('xAxis', 'index')
+        y_axis = request.query_params.get('yAxis', 'mean_va')
+        filter_param = request.query_params.get('filter', 'parameter_cd')
+        filter_vals = request.query_params.getlist('filterVals', ['00060', '00065'])
+        bounds = request.query_params.getlist(
+            'bounds', None
+        )  # Optional: [min_x, min_y, max_x, max_y]
+
+        if not table_type or not map_layer_id:
+            return Response(
+                {'error': 'tableType and mapLayerId are required'},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        try:
+            # Filter tables by map layer and table type
+            tables = VectorFeatureTableData.objects.filter(
+                type=table_type, map_layer_id=map_layer_id
+            )
+
+            if not tables.exists():
+                return Response(
+                    {'error': 'No tables found for the given parameters'},
+                    status=status.HTTP_404_NOT_FOUND,
+                )
+
+            table_data = {'map_layer_id': map_layer_id, 'graphs': {}}
+            all_x_values = set()
+
+            for table in tables:
+                table_entry = {'table_name': table.name}
+
+                x_axis_index = table.columns.index(x_axis)
+                y_axis_index = table.columns.index(y_axis)
+
+                # Prepare graph data for each filter value
+                if filter_param and len(filter_vals):
+                    filter_param_index = table.columns.index(filter_param)
+                    for filter_val in filter_vals:
+                        table_data['graphs'][filter_val] = {
+                            'data': [],
+                            'filterVal': filter_val,
+                        }
+                else:
+                    table_data['graphs']['default'] = {
+                        'data': [],
+                    }
+
+                rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
+
+                # If bounds are provided, filter VectorFeatures inside bounds
+                if bounds and len(bounds) == 4:
+                    min_x, min_y, max_x, max_y = map(float, bounds)
+                    bounds_polygon = Polygon.from_bbox((min_x, min_y, max_x, max_y))
+                    rows = rows.filter(
+                        vector_feature_table__vector_feature__geometry__intersects=bounds_polygon
+                    )
+
+                for row in rows:
+                    row_data = row.row_data
+                    x_val = row_data[x_axis_index]
+                    y_val = row_data[y_axis_index]
+                    all_x_values.add(x_val)
+
+                    # Handle filter logic for specific filter values
+                    if filter_param and len(filter_vals):
+                        filter_row = row_data[filter_param_index]
+                        for filter_val in filter_vals:
+                            if filter_row == filter_val:
+                                table_data['graphs'][filter_val]['data'].append([x_val, y_val])
+                    else:
+                        table_data['graphs']['default']['data'].append([x_val, y_val])
+
+                # Add the table data to the main response structure
+                table_data['tables'] = table_entry
+
+            # Sort and condense x-axis values into a single list
+            table_data['all_x_values'] = sorted(all_x_values)
+
+            return Response(table_data, status=status.HTTP_200_OK)
+
+        except Exception as e:
+            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
     @action(detail=False, methods=['post'], url_path='generate-layer')
-    @permission_classes([IsAuthenticated])
     def generate_layer(self, request, *args, **kwargs):
         table_id = request.data.get('table_id')
         filter_value = request.data.get('filter_value')
diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index 0c35c27..1f2e4c9 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -1,20 +1,20 @@
 import base64
 from datetime import datetime
-import cftime
 from io import BytesIO
 import logging
 from pathlib import Path
 import re
 import tempfile
-import pandas as pd
 
 from PIL import Image
 from celery import current_task, shared_task
+import cftime
 from django.contrib.gis.geos import Polygon
 from django.contrib.gis.geos.error import GEOSException
 from django.core.files.base import ContentFile
 from matplotlib import cm
 import numpy as np
+import pandas as pd
 import xarray as xr
 
 from uvdat.core.models import NetCDFData, NetCDFImage, NetCDFLayer, ProcessingTask
@@ -29,7 +29,15 @@ def convert_time(obj, output='compact'):
         dt_obj = pd.Timestamp(obj).to_pydatetime()
     elif isinstance(obj, datetime):  # Handle Python datetime objects
         dt_obj = obj
-    elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+    elif isinstance(
+        obj,
+        (
+            cftime.DatetimeNoLeap,
+            cftime.DatetimeAllLeap,
+            cftime.Datetime360Day,
+            cftime.DatetimeJulian,
+        ),
+    ):
         dt_obj = datetime(obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second)
     elif isinstance(obj, (int, float)):
         if obj > 1e10:  # Assume milliseconds timestamp
@@ -57,7 +65,15 @@ def convert_to_timestamp(obj):
         return (obj - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
     elif isinstance(obj, datetime):  # Handle Python datetime objects
         return obj.timestamp()
-    elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+    elif isinstance(
+        obj,
+        (
+            cftime.DatetimeNoLeap,
+            cftime.DatetimeAllLeap,
+            cftime.Datetime360Day,
+            cftime.DatetimeJulian,
+        ),
+    ):
         dt = obj
         dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
         unix_timestamp = pd.Timestamp(dt_obj).timestamp()
@@ -65,6 +81,7 @@ def convert_to_timestamp(obj):
     else:
         return obj
 
+
 def create_netcdf_data_layer(file_item, metadata):
     with tempfile.TemporaryDirectory() as temp_dir:
         raw_data_path = Path(temp_dir, 'netcdf.nc')
@@ -92,7 +109,13 @@ def create_netcdf_data_layer(file_item, metadata):
             }
 
             # Calculate min and max values if the variable has numeric data
-            if isinstance(variable.values[0], (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
+            cftime_types = (
+                cftime.DatetimeNoLeap,
+                cftime.DatetimeAllLeap,
+                cftime.Datetime360Day,
+                cftime.DatetimeJulian,
+            )
+            if isinstance(variable.values[0], cftime_types):
                 vals = []
                 for item in variable.values:
                     print(item)
@@ -115,7 +138,6 @@ def create_netcdf_data_layer(file_item, metadata):
                     if 'datetime' in str(variable.dtype):
                         var_info['startDate'] = str(variable.min().values)
                         var_info['endDate'] = str(variable.max().values)
-              
                     if 'time' in var_name:
                         var_info['min'] = convert_time(var_min, 'unix')
                         var_info['max'] = convert_time(var_max, 'unix')
@@ -141,7 +163,7 @@ def create_netcdf_data_layer(file_item, metadata):
                 except Exception:
                     var_info['min'] = 0
                     var_info['max'] = variable.size
-                    var_info["steps"] = variable.size
+                    var_info['steps'] = variable.size
 
             description['variables'][var_name] = var_info
 

From de1d42b62db602b85f5d15a7691cf1f832aee539 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 12 Feb 2025 13:11:29 -0500
Subject: [PATCH 09/16] linting

---
 .../FeatureSelection/VectorFeatureChart.vue   | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/client/src/components/FeatureSelection/VectorFeatureChart.vue b/client/src/components/FeatureSelection/VectorFeatureChart.vue
index a33f7ab..847cc35 100644
--- a/client/src/components/FeatureSelection/VectorFeatureChart.vue
+++ b/client/src/components/FeatureSelection/VectorFeatureChart.vue
@@ -29,25 +29,6 @@ export default defineComponent({
     const graphData = ref<FeatureGraphData | null>(null);
     const dialogVisible = ref(false);
 
-    // Fetch feature graph data when component is mounted or props change
-    const fetchFeatureGraphData = async () => {
-      try {
-        const data = await UVdatApi.getFeatureGraphData(
-          props.graphInfo.type, // Use graphInfo.type (tableType) instead of mapLayerId
-          props.vectorFeatureId,
-          props.graphInfo.xAxis,
-          props.graphInfo.yAxis,
-          props.graphInfo.filterColumn,
-          props.graphInfo.filterValue,
-        );
-        graphData.value = data;
-        renderGraph(data);
-      } catch (error) {
-        // eslint-disable-next-line no-console
-        console.error('Error fetching feature graph data:', error);
-      }
-    };
-
     // Render graph using D3
     const renderGraph = (data: FeatureGraphData, container = 'graphContainer') => {
       const localContainer = container === 'graphContainer' ? graphContainer : graphDialogContainer;
@@ -117,6 +98,25 @@ export default defineComponent({
         .call(d3.axisLeft(y));
     };
 
+    // Fetch feature graph data when component is mounted or props change
+    const fetchFeatureGraphData = async () => {
+      try {
+        const data = await UVdatApi.getFeatureGraphData(
+          props.graphInfo.type, // Use graphInfo.type (tableType) instead of mapLayerId
+          props.vectorFeatureId,
+          props.graphInfo.xAxis,
+          props.graphInfo.yAxis,
+          props.graphInfo.filterColumn,
+          props.graphInfo.filterValue,
+        );
+        graphData.value = data;
+        renderGraph(data);
+      } catch (error) {
+        // eslint-disable-next-line no-console
+        console.error('Error fetching feature graph data:', error);
+      }
+    };
+
     // Watch for prop changes and refetch data
     watch(
       () => [props.graphInfo, props.vectorFeatureId],

From 5b87c69bbbd6f400d7d9d9b733547a5966ce8242 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 12 Feb 2025 16:47:48 -0500
Subject: [PATCH 10/16] script update to flatten USGS meter data

---
 scripts/nwis/getStats.py | 126 +++++++++++++++++++++++++++++++++------
 1 file changed, 108 insertions(+), 18 deletions(-)

diff --git a/scripts/nwis/getStats.py b/scripts/nwis/getStats.py
index 9f3f0db..ce4d92f 100755
--- a/scripts/nwis/getStats.py
+++ b/scripts/nwis/getStats.py
@@ -4,7 +4,28 @@
 import pandas as pd
 from dataretrieval import nwis
 import numpy as np
+import time
+import calendar
 
+def get_unix_time(row, year_index, month_index, day_index):
+    if year_index is not None:
+        year = int(row[year_index])
+    else:
+        year = 1970
+    if month_index is not None:
+        month = int(row[month_index])
+    else:
+        month = 1
+    if day_index is not None:
+        day = int(row[day_index])
+    else:
+        day = 1
+    if month < 1 or month > 12:
+        month = 1
+    if day < 1 or day > 31:
+        day = 1
+
+    return calendar.timegm(time.strptime(f"{year}-{month}-{day}", "%Y-%m-%d"))
 # Define the function for fetching and formatting data
 @click.command()
 @click.option('-p', '--param-codes', multiple=True, required=True, help="List of parameter codes to query.")
@@ -42,13 +63,18 @@ def split_list(l):
     
     for report_type in report_types:
         for i, site_list in enumerate(site_lists):
-            response = nwis.get_stats(
-                sites=site_list,
-                startDt=start_date,
-                endDt=end_date,
-                statReportType=report_type,
-                parameterCd=param_codes,
-            )
+            try:
+                response = nwis.get_stats(
+                    sites=site_list,
+                    startDt=start_date,
+                    endDt=end_date,
+                    statReportType=report_type,
+                    parameterCd=",".join(param_codes),
+                )
+                df, meta = response
+            except Exception as e:
+                print(f"Error fetching {report_type} data for sites {site_list}: {e}")
+                continue
             df, meta = response
 
             # Replace NaN values with empty strings
@@ -59,19 +85,19 @@ def split_list(l):
                 site_data = df[df['site_no'] == site_number]
                 if not site_data.empty:
                     # Remove the 'ts_id' column if it exists
-                    site_data = site_data.drop(columns=['ts_id', 'site_no', 'loc_web_ds'], errors='ignore')
+                    site_data = site_data.drop(columns=['site_no', 'loc_web_ds'], errors='ignore')
 
                     unique_param_codes = site_data['parameter_cd'].unique()
-
                     # Create a description of the parameters
                     param_names = [param_desc.get(code, 'Unknown parameter') for code in unique_param_codes]
                     description = f"This is a table of the mean {report_type} values for the following parameters: {', '.join([f'{code} - {name}' for code, name in zip(unique_param_codes, param_names)])}"
-
                     header = site_data.columns.tolist()
+                    parameter_cd_index = header.index('parameter_cd')
+                    mean_va_index = header.index('mean_va')
 
                     # Prepare table object
                     rows = site_data.values.tolist()
-                    index = 0
+
                     base_set = set()
                     year_index = None
                     month_index = None
@@ -82,32 +108,82 @@ def split_list(l):
                         month_index = header.index('month_nu')
                     if 'day_nu' in header:
                         day_index = header.index('day_nu')
+                    base_string_param_map = {}
+                    ts_id_index = header.index('ts_id')
+                    # update to take only the latest ts_id
+                    ts_id_map = {}
                     for row in rows:
+                        ts_id = row[ts_id_index]
                         year = row[year_index] if year_index is not None else 0000
                         month = row[month_index] if month_index is not None else 00
                         day = row[day_index] if day_index is not None else 00
                         base_string = str(f'{str(year).zfill(4)}{str(month).zfill(2)}{str(day).zfill(2)}')
                         base_set.add(base_string)
+                        if base_string_param_map.get(base_string, None) is None:
+                            base_string_param_map[base_string] = {}
+                            for param in unique_param_codes:
+                                base_string_param_map[base_string][param] = None
+                        val = row[mean_va_index]
+                        for code in unique_param_codes:
+                            if row[parameter_cd_index] == code:
+                                if ts_id_map.get(f'{base_string}_{code}', None) is None:
+                                    ts_id_map[f'{base_string}_{code}'] = ts_id
+                                    if base_string_param_map[base_string][code] is None or val is not None:
+                                        base_string_param_map[base_string][code] = val
+                                if ts_id > ts_id_map[f'{base_string}_{code}']:
+                                    ts_id_map[f'{base_string}_{code}'] = ts_id
+                                    base_string_param_map[base_string][code] = val
                     base_order = list(base_set)
                     base_order.sort()
-                    print(base_order)
+                    time_base_index = {}
+                    header.append('index')
+                    header.append('unix_time')
+                    for code in unique_param_codes:
+                        header.append(code) 
+                    parameter_cd_index = header.index('parameter_cd')
+                    header.pop(parameter_cd_index)
+                    mean_va_index = header.index('mean_va')
+                    header.pop(mean_va_index)
+                    unix_mapping = {}
                     for row in rows:
                         year = row[year_index] if year_index is not None else 0000
                         month = row[month_index] if month_index is not None else 00
                         day = row[day_index] if day_index is not None else 00
                         base_string = str(f'{str(year).zfill(4)}{str(month).zfill(2)}{str(day).zfill(2)}')
                         row.append(base_order.index(base_string))
+                        unix_timestamp = get_unix_time(row, year_index, month_index, day_index)
+                        row.append(unix_timestamp)
+                        row.pop(parameter_cd_index)
+                        row.pop(mean_va_index)
+                        if time_base_index.get(base_string, None) is None:
+                            for param in unique_param_codes:
+                                param_val = base_string_param_map.get(base_string, {}).get(param, None)
+                                row.append(param_val)
+                        if unix_mapping.get(unix_timestamp, None) is None:
+                            unix_mapping[unix_timestamp] = row
+                        else:  # Combine the so we get rid of any missing data
+                            old_row = unix_mapping[unix_timestamp]
+                            for index in range(len(row)):
+                                if row[index] is not None and old_row[index] is None:
+                                    old_row[index] = row[index]
 
-                    header.append('index')
+                    # row_length = len(rows[0])
+                    # for row in rows:
+                    #     if len(row) != row_length:
+                    #         print(f'{site_number}  - {row} != {row_length}')
+                    #     else:
+                    #         print(f'{site_number}  - {row} == {row_length}')
+                    sorted_values = [value for _, value in sorted(unix_mapping.items())]
+
+                    updated_df = pd.DataFrame(sorted_values, columns=header)
                     table_object = {
                         "name": f"{site_number}_{report_type}",
                         "description": description,
                         "type": f'USGS_gauge_{report_type}_{"_".join(param_codes)}',
                         "header": header,
-                        "summary": generate_summary(site_data, param_desc, site_data.columns.tolist()),
-                        "rows": rows,
+                        "summary": generate_summary(updated_df, param_desc, updated_df.columns.tolist()),
+                        "rows": sorted_values,
                     }
-
                     if site_number not in result:
                         result[site_number] = []
                     
@@ -145,8 +221,12 @@ def generate_summary(df, param_desc, rows):
                     "mean": float(value_col.mean())
                 }
         else:  # Calculate type/min/max and other fields
+
             if header not in summary.keys():
                 summary[header] = {'type': None, 'values': set(), 'value_count': 0}
+                parameter_cd = param_desc.get(header, None)
+                if parameter_cd:
+                    summary[header]["description"] = parameter_cd,
             for value in df[header].unique():
                 if isinstance(value, bool):
                     summary[header]['type'] = 'bool'
@@ -155,9 +235,15 @@ def generate_summary(df, param_desc, rows):
                     summary[header]['type'] = 'number'
                     summary[header]['value_count'] += 1
                     if 'min' not in summary[header] or value < summary[header]['min']:
-                        summary[header]['min'] = float(value)
+                        if np.isnan(float(value)) or value is None and summary[header].get('min', None) is None:
+                            summary[header]['min'] = float('inf')
+                        else:
+                            summary[header]['min'] = float(value)
                     if 'max' not in summary[header] or value > summary[header]['max']:
-                        summary[header]['max'] = float(value)
+                        if np.isnan(float(value)) or value is None and summary[header].get('max', None) is None:
+                            summary[header]['max'] = float('-inf')
+                        else:
+                            summary[header]['max'] = float(value)
                 elif isinstance(value, str):
                     if 'values' not in summary[header]:
                         summary[header]['values'] = set()
@@ -182,6 +268,10 @@ def generate_summary(df, param_desc, rows):
                 summary[header]['static'] = True
                 summary[header]['value'] = val
             else:
+                if np.isnan(summary[header]['min']):
+                    summary[header]['min'] = None
+                if np.isnan(summary[header]['max']):
+                    summary[header]['max'] = None
                 del summary[header]['values']
         elif (
             summary[header]['type'] == 'string'

From e1e873ff2deeae335604a62878ffdac8f194f4a3 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 12 Feb 2025 20:26:37 -0500
Subject: [PATCH 11/16] update table structure

---
 .../FeatureSelection/VectorFeatureChart.vue   |   7 +-
 .../components/TabularData/TableSummary.vue   |  48 ++---
 client/src/types.ts                           |  27 +--
 scripts/nwis/getTVASites.py                   |   2 +-
 uvdat/core/rest/vector_feature_table_data.py  | 193 +++++-------------
 5 files changed, 84 insertions(+), 193 deletions(-)

diff --git a/client/src/components/FeatureSelection/VectorFeatureChart.vue b/client/src/components/FeatureSelection/VectorFeatureChart.vue
index 847cc35..ecdf7e9 100644
--- a/client/src/components/FeatureSelection/VectorFeatureChart.vue
+++ b/client/src/components/FeatureSelection/VectorFeatureChart.vue
@@ -55,11 +55,8 @@ export default defineComponent({
       let dataForGraph: { data: [number, number][], filterVal?: string } | undefined;
 
       // Check for default data or apply filter if necessary
-      if (data.graphs.default) {
-        dataForGraph = data.graphs.default; // Use default data if no filter
-      } else if (props.graphInfo.filterValue && props.graphInfo.filterColumn) {
-        const filteredData = data.graphs as ParameterGraph;
-        dataForGraph = filteredData[props.graphInfo.filterValue]; // Apply filter by value
+      if (data.graphs[props.vectorFeatureId]) {
+        dataForGraph = data.graphs[props.vectorFeatureId]
       }
 
       if (!dataForGraph) {
diff --git a/client/src/components/TabularData/TableSummary.vue b/client/src/components/TabularData/TableSummary.vue
index 4a66b47..65a08fb 100644
--- a/client/src/components/TabularData/TableSummary.vue
+++ b/client/src/components/TabularData/TableSummary.vue
@@ -1,6 +1,6 @@
 <script lang="ts">
 import {
-  Ref, defineComponent, onMounted, ref,
+  Ref, defineComponent, onMounted, ref, computed,
 } from 'vue';
 import { TableSummary, VectorFeatureTableGraph } from '../../types';
 import UVdatApi from '../../api/UVDATApi';
@@ -24,8 +24,7 @@ export default defineComponent({
     const selectedTableType = ref<string | null>(null);
     const selectedXColumn = ref<string | null>(null);
     const selectedYColumn = ref<string | null>(null);
-    const selectedFilterColumn = ref<string>('');
-    const filterValue = ref<string>('');
+    const selectedIndexerColumn = ref<string>('');
     const graphs = ref<VectorFeatureTableGraph[]>([]);
 
     const getStyleVectorFeatureGraphs = () => {
@@ -59,8 +58,7 @@ export default defineComponent({
     const resetGraphForm = () => {
       selectedXColumn.value = null;
       selectedYColumn.value = null;
-      selectedFilterColumn.value = '';
-      filterValue.value = '';
+      selectedIndexerColumn.value = '';
     };
 
     const addSaveGraph = () => {
@@ -70,8 +68,7 @@ export default defineComponent({
           type: selectedTableType.value,
           xAxis: selectedXColumn.value,
           yAxis: selectedYColumn.value,
-          filterColumn: selectedFilterColumn.value ? selectedFilterColumn.value : undefined,
-          filterValue: filterValue.value ? filterValue.value : undefined,
+          indexer: selectedIndexerColumn.value ? selectedIndexerColumn.value : undefined,
         };
 
         if (editingGraphIndex.value !== null) {
@@ -89,6 +86,15 @@ export default defineComponent({
       }
     };
 
+    const availableColumns = computed(() => {
+      const baseColumns = selectedTableType ? summaryData?.tables[selectedTableType]?.columns : [];
+      if (baseColumns.length) {
+        const takenVals = [selectedXColumn.value, selectedYColumn.value, selectedIndexerColumn].filter((item) => item !== null);
+        return baseColumns.filter((item) => !takenVals.includes(item))
+      }
+      return baseColumns;
+    })
+
     const editGraph = (index: number) => {
       const graph = graphs.value[index];
       // eslint-disable-next-line prefer-destructuring
@@ -96,8 +102,7 @@ export default defineComponent({
       selectedTableName.value = graph.name;
       selectedXColumn.value = graph.xAxis;
       selectedYColumn.value = graph.yAxis;
-      selectedFilterColumn.value = graph.filterColumn || '';
-      filterValue.value = graph.filterValue || ' ';
+      selectedIndexerColumn.value = graph.indexer || '';
       editingGraphIndex.value = index;
     };
 
@@ -116,13 +121,13 @@ export default defineComponent({
       error,
       tableChartDialog,
       activeTab,
+      availableColumns,
       selectedTableName,
       selectedTableType,
       selectedXColumn,
       selectedYColumn,
-      selectedFilterColumn,
+      selectedIndexerColumn,
       editingGraphIndex,
-      filterValue,
       graphs,
       addSaveGraph,
       editGraph,
@@ -203,23 +208,18 @@ export default defineComponent({
             />
             <v-select
               v-model="selectedXColumn"
-              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
+              :items="availableColumns"
               label="Select X Axis"
             />
             <v-select
               v-model="selectedYColumn"
-              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
+              :items="availableColumns"
               label="Select Y Axis"
             />
             <v-select
-              v-model="selectedFilterColumn"
-              :items="selectedTableType ? summaryData?.tables[selectedTableType]?.columns : []"
-              label="Select Filter Column"
-            />
-            <v-text-field
-              v-model="filterValue"
-              label="Filter Value"
-              :disabled="!selectedFilterColumn"
+              v-model="selectedIndexerColumn"
+              :items="availableColumns"
+              label="Select Indexer Column"
             />
             <v-btn :disabled="!selectedTableType || !selectedXColumn || !selectedYColumn" color="primary" @click="addSaveGraph">
               {{ editingGraphIndex !== null ? 'Update' : 'Add' }} Graph
@@ -231,10 +231,10 @@ export default defineComponent({
               <v-list-item-title>{{ graph.name }}</v-list-item-title>
               <v-list-item-subtitle>
                 <div>{{ graph.type }}</div>
-                <div>{{ graph.xAxis }} vs {{ graph.yAxis }} {{ graph.filterValue ? `(Filter - ${graph.filterValue}` : '' }})</div>
+                <div>{{ graph.xAxis }} vs {{ graph.yAxis }} {{ graph.indexer ? `(Indexer - ${graph.indexer}` : '' }})</div>
               </v-list-item-subtitle>
-              <v-list-item-subtitle v-if="graph.filterColumn">
-                Filter: {{ graph.filterColumn }} = {{ graph.filterValue }}
+              <v-list-item-subtitle v-if="graph.inexer">
+                Indexer: {{ graph.indexer }}
               </v-list-item-subtitle>
               <v-list-item-action>
                 <v-icon color="warning" @click="editGraph(index)">
diff --git a/client/src/types.ts b/client/src/types.ts
index 20587c4..0d21e85 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -750,53 +750,38 @@ interface TableInfo {
 type ColumnSummary =
   | NumberColumnSummary
   | StringColumnSummary
-  | ParameterColumnSummary;
 
 interface NumberColumnSummary {
   type: 'number';
   min: number;
   max: number;
   value_count: number;
+  description?: string;
 }
 
 interface StringColumnSummary {
   type: 'string';
   values: string[];
   value_count: number;
+  description?: string;
 }
 
-interface ParameterColumnSummary {
-  type: 'parameter_cd';
-  parameters: Record<string, ParameterStats>;
-}
 
-interface ParameterStats {
-  parameter_cd: string;
-  parameter_name: string;
-  min: number;
-  max: number;
-  mean: number;
-}
 
 export interface VectorFeatureTableGraph {
   name: string;
   type: string;
   xAxis: string;
   yAxis: string;
-  filterColumn?: string;
-  filterValue?: string;
+  indexer: string;
 }
 
 export interface FeatureGraphData {
   table_name: string;
   graphs:
-  Record<string, {
+  Record<number, {
     data:[number, number][];
-    filterVal: string;
+    vectorFeatureId: number;
+    indexer: string | number;
   }>
-  | {
-    default: {
-      data: [number, number][];
-    };
-  };
 }
diff --git a/scripts/nwis/getTVASites.py b/scripts/nwis/getTVASites.py
index 6c1c24f..97d241a 100755
--- a/scripts/nwis/getTVASites.py
+++ b/scripts/nwis/getTVASites.py
@@ -68,7 +68,7 @@ def fetch_tva_sites(param_codes, usgs_parameters, site_types):
         print(f"ParamCodes: {param_codes}")
         site_response = nwis.what_sites(
             bBox=bbox,
-            parameterCd=list(param_codes),
+            parameterCd=",".join(param_codes),
             siteType=query_site_types,
             siteStatus='active',
             startDt=start_dt,
diff --git a/uvdat/core/rest/vector_feature_table_data.py b/uvdat/core/rest/vector_feature_table_data.py
index 5bba118..73e20cb 100644
--- a/uvdat/core/rest/vector_feature_table_data.py
+++ b/uvdat/core/rest/vector_feature_table_data.py
@@ -132,36 +132,8 @@ def table_summary(self, request, *args, **kwargs):
                         table_type
                     ][column].get('value_count', 0) + stats.get('value_count', 0)
 
-                # Special USGS Parameter Type
-                elif stats.get('type') == 'parameter_cd':
-                    if 'parameters' not in column_summaries[table_type][column]:
-                        column_summaries[table_type][column]['parameters'] = {}
-
-                    for param_cd, param_stats in stats.items():
-                        if param_cd == 'type':
-                            continue
-                        if param_cd not in column_summaries[table_type][column]['parameters']:
-                            column_summaries[table_type][column]['parameters'][
-                                param_cd
-                            ] = param_stats
-                        else:
-                            column_summaries[table_type][column]['parameters'][param_cd]['min'] = (
-                                min(
-                                    column_summaries[table_type][column]['parameters'][param_cd][
-                                        'min'
-                                    ],
-                                    param_stats['min'],
-                                )
-                            )
-                            column_summaries[table_type][column]['parameters'][param_cd]['max'] = (
-                                max(
-                                    column_summaries[table_type][column]['parameters'][param_cd][
-                                        'max'
-                                    ],
-                                    param_stats['max'],
-                                )
-                            )
-
+                if stats.get('description', None):
+                    column_summaries[table_type][column]['desciption'] = stats.get('description', 'Unknown')
         # Construct the response
         output = {'vectorFeatureCount': feature_count, 'tables': {}}
         for table_type in type_columns_map:
@@ -174,49 +146,46 @@ def table_summary(self, request, *args, **kwargs):
 
         return Response(output, status=status.HTTP_200_OK)
 
-    @action(detail=False, methods=['get'], url_path='feature-graph')
-    def feature_graph(self, request, *args, **kwargs):
-        table_type = request.query_params.get('tableType')  # array of Ids
-        vector_feature = request.query_params.get('vectorFeatureId')
-        x_axis = request.query_params.get('xAxis', 'index')
-        y_axis = request.query_params.get('yAxis', 'mean_va')
-        filter_param = request.query_params.get('filter', 'parameter_cd')
-        filter_val = request.query_params.get('filterVal', '00065')
-        logger.info(f'filter_vals: {filter_val}')
-        if not table_type:
-            return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
-
-        try:
-            table = VectorFeatureTableData.objects.get(
-                type=table_type, vector_feature=vector_feature
-            )
-            table_data = {'table_name': table.name}
+    def get_graphs(table_type, vector_ids, x_axis, y_axis, indexer='vectorFeatureId'):
+        tables = VectorFeatureTableData.objects.filter(
+            type=table_type, vector_feature__in=vector_ids
+        )
+        table_data = {'tableName': table.name, 'graphs': {}}
+        for table in tables:
             x_axis_index = table.columns.index(x_axis)
             y_axis_index = table.columns.index(y_axis)
-            if filter_param and filter_val:
-                filter_param_index = table.columns.index(filter_param)
-            if filter_param and filter_val:
-                table_data['graphs'] = {}
-                table_data['graphs'][filter_val] = {
-                    'data': [],
-                    'filterVal': filter_val,
-                }
+            vector_feature_id = table.vector_feature.pk
+            if indexer == 'vectorFeatureId':
+                index_val = vector_feature_id
             else:
-                table_data['graphs'] = {'default': {'data': []}}
+                index_val = table.vector_feature.properties.get(indexer)
+            table_data['graphs'][vector_feature_id] = {
+                'indexer': index_val,
+                'vectorFeatureId': vector_feature_id,
+                'data' : []
+             }
             rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
             for row in rows:
                 row_data = row.row_data
-                filter_row = row_data[filter_param_index]
                 x_val = row_data[x_axis_index]
                 y_val = row_data[y_axis_index]
-                if filter_param and filter_val:
-                    if filter_row == filter_val:
-                        table_data['graphs'][filter_val]['data'].append([x_val, y_val])
-                else:
-                    table_data['graphs']['default']['data'].append([x_val, y_val])
-            return Response(table_data, status=status.HTTP_200_OK)
-        except VectorFeatureTableData.DoesNotExist:
-            return Response({'error': 'Table not found'}, status=status.HTTP_404_NOT_FOUND)
+                table_data['graphs'][vector_feature_id]['data'].append([x_val, y_val])
+        return table_data
+
+
+
+    @action(detail=False, methods=['get'], url_path='feature-graph')
+    def feature_graph(self, request, *args, **kwargs):
+        table_type = request.query_params.get('tableType')  # array of Ids
+        vector_feature = request.query_params.get('vectorFeatureId')
+        x_axis = request.query_params.get('xAxis', 'index')
+        y_axis = request.query_params.get('yAxis', '00060')
+        indexer = request.query_params.get('indexer', 'vectorFeatureId')
+        if not table_type:
+            return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
+
+        graphs = self.get_graphs([vector_feature], x_axis, y_axis, indexer)
+        return Response(graphs, status=status.HTTP_200_OK)
 
     @action(detail=False, methods=['get'], url_path='map-layer-feature-graph')
     def map_layer_feature_graph(self, request, *args, **kwargs):
@@ -224,87 +193,27 @@ def map_layer_feature_graph(self, request, *args, **kwargs):
         map_layer_id = request.query_params.get('mapLayerId')  # Required
         x_axis = request.query_params.get('xAxis', 'index')
         y_axis = request.query_params.get('yAxis', 'mean_va')
-        filter_param = request.query_params.get('filter', 'parameter_cd')
-        filter_vals = request.query_params.getlist('filterVals', ['00060', '00065'])
-        bounds = request.query_params.getlist(
-            'bounds', None
+        indexer = request.query_params.get('indexer', 'vectorFeatureId')
+        bbox = request.query_params.getlist(
+            'bbox', None
         )  # Optional: [min_x, min_y, max_x, max_y]
-
-        if not table_type or not map_layer_id:
-            return Response(
-                {'error': 'tableType and mapLayerId are required'},
-                status=status.HTTP_400_BAD_REQUEST,
-            )
-
+        
+        if not bbox:
+            return Response({'error': 'bbox parameter is required'}, status=400)
+        
         try:
-            # Filter tables by map layer and table type
-            tables = VectorFeatureTableData.objects.filter(
-                type=table_type, map_layer_id=map_layer_id
-            )
-
-            if not tables.exists():
-                return Response(
-                    {'error': 'No tables found for the given parameters'},
-                    status=status.HTTP_404_NOT_FOUND,
-                )
-
-            table_data = {'map_layer_id': map_layer_id, 'graphs': {}}
-            all_x_values = set()
-
-            for table in tables:
-                table_entry = {'table_name': table.name}
+            xmin, ymin, xmax, ymax = map(float, bbox.split(","))
+            bbox_polygon = Polygon.from_bbox((xmin, ymin, xmax, ymax))
+        except ValueError:
+            return Response({"error": "Invalid bbox format. Expected format: xmin,ymin,xmax,ymax"}, status=400)
+        
+        vector_features = VectorFeature.objects.filter(geometry__intersects=bbox_polygon, map_layer=map_layer_id).values_list("id", flat=True)
 
-                x_axis_index = table.columns.index(x_axis)
-                y_axis_index = table.columns.index(y_axis)
-
-                # Prepare graph data for each filter value
-                if filter_param and len(filter_vals):
-                    filter_param_index = table.columns.index(filter_param)
-                    for filter_val in filter_vals:
-                        table_data['graphs'][filter_val] = {
-                            'data': [],
-                            'filterVal': filter_val,
-                        }
-                else:
-                    table_data['graphs']['default'] = {
-                        'data': [],
-                    }
-
-                rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
-
-                # If bounds are provided, filter VectorFeatures inside bounds
-                if bounds and len(bounds) == 4:
-                    min_x, min_y, max_x, max_y = map(float, bounds)
-                    bounds_polygon = Polygon.from_bbox((min_x, min_y, max_x, max_y))
-                    rows = rows.filter(
-                        vector_feature_table__vector_feature__geometry__intersects=bounds_polygon
-                    )
-
-                for row in rows:
-                    row_data = row.row_data
-                    x_val = row_data[x_axis_index]
-                    y_val = row_data[y_axis_index]
-                    all_x_values.add(x_val)
-
-                    # Handle filter logic for specific filter values
-                    if filter_param and len(filter_vals):
-                        filter_row = row_data[filter_param_index]
-                        for filter_val in filter_vals:
-                            if filter_row == filter_val:
-                                table_data['graphs'][filter_val]['data'].append([x_val, y_val])
-                    else:
-                        table_data['graphs']['default']['data'].append([x_val, y_val])
-
-                # Add the table data to the main response structure
-                table_data['tables'] = table_entry
-
-            # Sort and condense x-axis values into a single list
-            table_data['all_x_values'] = sorted(all_x_values)
-
-            return Response(table_data, status=status.HTTP_200_OK)
+        if not table_type:
+            return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
 
-        except Exception as e:
-            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        graphs = self.get_graphs(vector_features, x_axis, y_axis, indexer)
+        return Response(graphs, status=status.HTTP_200_OK)
 
     @action(detail=False, methods=['post'], url_path='generate-layer')
     def generate_layer(self, request, *args, **kwargs):

From df10de6f1bd9fd04c201e3a263c6b7796b035e90 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 13 Feb 2025 08:41:31 -0500
Subject: [PATCH 12/16] update vectorfeature graphing

---
 .../components/TabularData/TableSummary.vue   | 31 +++++++++++++++----
 client/src/types.ts                           |  2 +-
 scripts/nwis/getStats.py                      |  2 +-
 uvdat/core/rest/vector_feature_table_data.py  | 28 ++++++++++-------
 4 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/client/src/components/TabularData/TableSummary.vue b/client/src/components/TabularData/TableSummary.vue
index 65a08fb..9a1d31c 100644
--- a/client/src/components/TabularData/TableSummary.vue
+++ b/client/src/components/TabularData/TableSummary.vue
@@ -1,6 +1,6 @@
 <script lang="ts">
 import {
-  Ref, defineComponent, onMounted, ref, computed,
+  Ref, computed, defineComponent, onMounted, ref,
 } from 'vue';
 import { TableSummary, VectorFeatureTableGraph } from '../../types';
 import UVdatApi from '../../api/UVDATApi';
@@ -87,13 +87,29 @@ export default defineComponent({
     };
 
     const availableColumns = computed(() => {
-      const baseColumns = selectedTableType ? summaryData?.tables[selectedTableType]?.columns : [];
-      if (baseColumns.length) {
+      const baseColumns = selectedTableType.value && summaryData.value?.tables[selectedTableType.value]?.columns
+        ? summaryData.value?.tables[selectedTableType.value]?.columns : [];
+      const mappedColumns = baseColumns.map((item) => {
+        const summary = selectedTableType.value && summaryData.value?.tables[selectedTableType.value].summary[item];
+        let subtitle = '';
+        if (summary && summary.description) {
+          subtitle = summary.description;
+        } else if (summary && summary.type === 'number') {
+          subtitle = `${summary.min} to ${summary.max}`;
+        } else if (summary && summary.type === 'string') {
+          subtitle = `${summary.value_count} unique values`;
+        }
+        return {
+          title: item,
+          subtitle,
+        };
+      });
+      if (mappedColumns.length) {
         const takenVals = [selectedXColumn.value, selectedYColumn.value, selectedIndexerColumn].filter((item) => item !== null);
-        return baseColumns.filter((item) => !takenVals.includes(item))
+        return mappedColumns.filter((item) => !takenVals.includes(item.title));
       }
-      return baseColumns;
-    })
+      return mappedColumns;
+    });
 
     const editGraph = (index: number) => {
       const graph = graphs.value[index];
@@ -209,16 +225,19 @@ export default defineComponent({
             <v-select
               v-model="selectedXColumn"
               :items="availableColumns"
+              :item-props="true"
               label="Select X Axis"
             />
             <v-select
               v-model="selectedYColumn"
               :items="availableColumns"
+              :item-props="true"
               label="Select Y Axis"
             />
             <v-select
               v-model="selectedIndexerColumn"
               :items="availableColumns"
+              :item-props="true"
               label="Select Indexer Column"
             />
             <v-btn :disabled="!selectedTableType || !selectedXColumn || !selectedYColumn" color="primary" @click="addSaveGraph">
diff --git a/client/src/types.ts b/client/src/types.ts
index 0d21e85..856adff 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -749,7 +749,7 @@ interface TableInfo {
 
 type ColumnSummary =
   | NumberColumnSummary
-  | StringColumnSummary
+  | StringColumnSummary;
 
 interface NumberColumnSummary {
   type: 'number';
diff --git a/scripts/nwis/getStats.py b/scripts/nwis/getStats.py
index ce4d92f..5e2f097 100755
--- a/scripts/nwis/getStats.py
+++ b/scripts/nwis/getStats.py
@@ -226,7 +226,7 @@ def generate_summary(df, param_desc, rows):
                 summary[header] = {'type': None, 'values': set(), 'value_count': 0}
                 parameter_cd = param_desc.get(header, None)
                 if parameter_cd:
-                    summary[header]["description"] = parameter_cd,
+                    summary[header]["description"] = parameter_cd[0] if isinstance(parameter_cd, tuple) else parameter_cd
             for value in df[header].unique():
                 if isinstance(value, bool):
                     summary[header]['type'] = 'bool'
diff --git a/uvdat/core/rest/vector_feature_table_data.py b/uvdat/core/rest/vector_feature_table_data.py
index 73e20cb..b40b840 100644
--- a/uvdat/core/rest/vector_feature_table_data.py
+++ b/uvdat/core/rest/vector_feature_table_data.py
@@ -97,10 +97,8 @@ def table_summary(self, request, *args, **kwargs):
             # Ensure all tables of the same type have consistent columns
             if len(type_columns_map[table_type]) == 0:
                 type_columns_map[table_type] = set(table.columns)
-            elif type_columns_map[table_type] != set(table.columns):
-                raise ValidationError(
-                    f'Table Columns do not match for all tables: {type_columns_map[table_type]} vs {set(table.columns)}'
-                )
+            for column in table.columns:
+                type_columns_map[table_type].add(column)
 
             table_count_map[table_type] += 1
 
@@ -133,7 +131,7 @@ def table_summary(self, request, *args, **kwargs):
                     ][column].get('value_count', 0) + stats.get('value_count', 0)
 
                 if stats.get('description', None):
-                    column_summaries[table_type][column]['desciption'] = stats.get('description', 'Unknown')
+                    column_summaries[table_type][column]['description'] = stats.get('description', 'Unknown')
         # Construct the response
         output = {'vectorFeatureCount': feature_count, 'tables': {}}
         for table_type in type_columns_map:
@@ -146,12 +144,19 @@ def table_summary(self, request, *args, **kwargs):
 
         return Response(output, status=status.HTTP_200_OK)
 
-    def get_graphs(table_type, vector_ids, x_axis, y_axis, indexer='vectorFeatureId'):
+    def get_graphs(self, table_type, vector_ids, x_axis, y_axis, indexer='vectorFeatureId'):
         tables = VectorFeatureTableData.objects.filter(
             type=table_type, vector_feature__in=vector_ids
         )
-        table_data = {'tableName': table.name, 'graphs': {}}
+        if tables.count() == 0:
+            return {'error': f'No tables found for the given vector features {table_type} - {vector_ids}'}
+        table_data = {'tableName': tables.first().name, 'graphs': {}}
         for table in tables:
+            if y_axis not in table.columns or x_axis not in table.columns:
+                logger.warning(
+                    f'Columns {x_axis} and {y_axis} not found in table {table.name} with id: {table.pk}'
+                )
+                continue
             x_axis_index = table.columns.index(x_axis)
             y_axis_index = table.columns.index(y_axis)
             vector_feature_id = table.vector_feature.pk
@@ -163,13 +168,14 @@ def get_graphs(table_type, vector_ids, x_axis, y_axis, indexer='vectorFeatureId'
                 'indexer': index_val,
                 'vectorFeatureId': vector_feature_id,
                 'data' : []
-             }
+            }
             rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
             for row in rows:
                 row_data = row.row_data
                 x_val = row_data[x_axis_index]
                 y_val = row_data[y_axis_index]
-                table_data['graphs'][vector_feature_id]['data'].append([x_val, y_val])
+                if y_val is not None:
+                    table_data['graphs'][vector_feature_id]['data'].append([x_val, y_val])
         return table_data
 
 
@@ -184,7 +190,7 @@ def feature_graph(self, request, *args, **kwargs):
         if not table_type:
             return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
 
-        graphs = self.get_graphs([vector_feature], x_axis, y_axis, indexer)
+        graphs = self.get_graphs(table_type, [vector_feature], x_axis, y_axis, indexer)
         return Response(graphs, status=status.HTTP_200_OK)
 
     @action(detail=False, methods=['get'], url_path='map-layer-feature-graph')
@@ -212,7 +218,7 @@ def map_layer_feature_graph(self, request, *args, **kwargs):
         if not table_type:
             return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)
 
-        graphs = self.get_graphs(vector_features, x_axis, y_axis, indexer)
+        graphs = self.get_graphs(table_type, vector_features, x_axis, y_axis, indexer)
         return Response(graphs, status=status.HTTP_200_OK)
 
     @action(detail=False, methods=['post'], url_path='generate-layer')

From fc8fcba7805904ad64a9b3c754723222a0e3ac39 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 13 Feb 2025 08:49:33 -0500
Subject: [PATCH 13/16] linting

---
 .../FeatureSelection/VectorFeatureChart.vue   |  7 +---
 client/src/types.ts                           |  2 --
 uvdat/core/rest/vector_feature_table_data.py  | 33 ++++++++++---------
 3 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/client/src/components/FeatureSelection/VectorFeatureChart.vue b/client/src/components/FeatureSelection/VectorFeatureChart.vue
index ecdf7e9..b1ccc47 100644
--- a/client/src/components/FeatureSelection/VectorFeatureChart.vue
+++ b/client/src/components/FeatureSelection/VectorFeatureChart.vue
@@ -6,11 +6,6 @@ import * as d3 from 'd3';
 import UVdatApi from '../../api/UVDATApi';
 import { FeatureGraphData, VectorFeatureTableGraph } from '../../types';
 
-type ParameterGraph = Record<string, {
-  data: [number, number][];
-  filterVal: string;
-}>;
-
 export default defineComponent({
   name: 'FeatureGraph',
   props: {
@@ -56,7 +51,7 @@ export default defineComponent({
 
       // Check for default data or apply filter if necessary
       if (data.graphs[props.vectorFeatureId]) {
-        dataForGraph = data.graphs[props.vectorFeatureId]
+        dataForGraph = data.graphs[props.vectorFeatureId];
       }
 
       if (!dataForGraph) {
diff --git a/client/src/types.ts b/client/src/types.ts
index 856adff..408ba0f 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -766,8 +766,6 @@ interface StringColumnSummary {
   description?: string;
 }
 
-
-
 export interface VectorFeatureTableGraph {
   name: string;
   type: string;
diff --git a/uvdat/core/rest/vector_feature_table_data.py b/uvdat/core/rest/vector_feature_table_data.py
index b40b840..882d659 100644
--- a/uvdat/core/rest/vector_feature_table_data.py
+++ b/uvdat/core/rest/vector_feature_table_data.py
@@ -6,7 +6,6 @@
 from django.http import HttpResponse
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
-from rest_framework.exceptions import ValidationError
 from rest_framework.response import Response
 
 from uvdat.core.models import VectorFeature, VectorFeatureRowData, VectorFeatureTableData
@@ -131,7 +130,9 @@ def table_summary(self, request, *args, **kwargs):
                     ][column].get('value_count', 0) + stats.get('value_count', 0)
 
                 if stats.get('description', None):
-                    column_summaries[table_type][column]['description'] = stats.get('description', 'Unknown')
+                    column_summaries[table_type][column]['description'] = stats.get(
+                        'description', 'Unknown'
+                    )
         # Construct the response
         output = {'vectorFeatureCount': feature_count, 'tables': {}}
         for table_type in type_columns_map:
@@ -149,7 +150,9 @@ def get_graphs(self, table_type, vector_ids, x_axis, y_axis, indexer='vectorFeat
             type=table_type, vector_feature__in=vector_ids
         )
         if tables.count() == 0:
-            return {'error': f'No tables found for the given vector features {table_type} - {vector_ids}'}
+            return {
+                'error': f'No tables found for the given vector features {table_type} - {vector_ids}'
+            }
         table_data = {'tableName': tables.first().name, 'graphs': {}}
         for table in tables:
             if y_axis not in table.columns or x_axis not in table.columns:
@@ -167,7 +170,7 @@ def get_graphs(self, table_type, vector_ids, x_axis, y_axis, indexer='vectorFeat
             table_data['graphs'][vector_feature_id] = {
                 'indexer': index_val,
                 'vectorFeatureId': vector_feature_id,
-                'data' : []
+                'data': [],
             }
             rows = VectorFeatureRowData.objects.filter(vector_feature_table=table)
             for row in rows:
@@ -178,8 +181,6 @@ def get_graphs(self, table_type, vector_ids, x_axis, y_axis, indexer='vectorFeat
                     table_data['graphs'][vector_feature_id]['data'].append([x_val, y_val])
         return table_data
 
-
-
     @action(detail=False, methods=['get'], url_path='feature-graph')
     def feature_graph(self, request, *args, **kwargs):
         table_type = request.query_params.get('tableType')  # array of Ids
@@ -200,20 +201,22 @@ def map_layer_feature_graph(self, request, *args, **kwargs):
         x_axis = request.query_params.get('xAxis', 'index')
         y_axis = request.query_params.get('yAxis', 'mean_va')
         indexer = request.query_params.get('indexer', 'vectorFeatureId')
-        bbox = request.query_params.getlist(
-            'bbox', None
-        )  # Optional: [min_x, min_y, max_x, max_y]
-        
+        bbox = request.query_params.getlist('bbox', None)  # Optional: [min_x, min_y, max_x, max_y]
+
         if not bbox:
             return Response({'error': 'bbox parameter is required'}, status=400)
-        
+
         try:
-            xmin, ymin, xmax, ymax = map(float, bbox.split(","))
+            xmin, ymin, xmax, ymax = map(float, bbox.split(','))
             bbox_polygon = Polygon.from_bbox((xmin, ymin, xmax, ymax))
         except ValueError:
-            return Response({"error": "Invalid bbox format. Expected format: xmin,ymin,xmax,ymax"}, status=400)
-        
-        vector_features = VectorFeature.objects.filter(geometry__intersects=bbox_polygon, map_layer=map_layer_id).values_list("id", flat=True)
+            return Response(
+                {'error': 'Invalid bbox format. Expected format: xmin,ymin,xmax,ymax'}, status=400
+            )
+
+        vector_features = VectorFeature.objects.filter(
+            geometry__intersects=bbox_polygon, map_layer=map_layer_id
+        ).values_list('id', flat=True)
 
         if not table_type:
             return Response({'error': 'tableType is required'}, status=status.HTTP_400_BAD_REQUEST)

From 81207989e52f917746a7e0b1acba71ba4ff9b651 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 13 Feb 2025 09:11:24 -0500
Subject: [PATCH 14/16] fix netcdf test ingestion

---
 uvdat/core/tasks/netcdf.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index 1f2e4c9..73d35e8 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -115,10 +115,9 @@ def create_netcdf_data_layer(file_item, metadata):
                 cftime.Datetime360Day,
                 cftime.DatetimeJulian,
             )
-            if isinstance(variable.values[0], cftime_types):
+            if variable.size > 0 and variable.values.ndim > 0 and isinstance(variable.values[0], cftime_types):
                 vals = []
                 for item in variable.values:
-                    print(item)
                     dt = item
                     dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
                     timeobj = convert_time(pd.Timestamp(dt_obj), 'datetime')

From 9acc782bfacd27e784850e8435bb764cf259bb77 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 13 Feb 2025 09:17:31 -0500
Subject: [PATCH 15/16] linting

---
 uvdat/core/tasks/netcdf.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/uvdat/core/tasks/netcdf.py b/uvdat/core/tasks/netcdf.py
index 73d35e8..16b0611 100644
--- a/uvdat/core/tasks/netcdf.py
+++ b/uvdat/core/tasks/netcdf.py
@@ -115,7 +115,11 @@ def create_netcdf_data_layer(file_item, metadata):
                 cftime.Datetime360Day,
                 cftime.DatetimeJulian,
             )
-            if variable.size > 0 and variable.values.ndim > 0 and isinstance(variable.values[0], cftime_types):
+            if (
+                variable.size > 0
+                and variable.values.ndim > 0
+                and isinstance(variable.values[0], cftime_types)
+            ):
                 vals = []
                 for item in variable.values:
                     dt = item

From 833880c4149b46709783ed2947db54eff08c5b69 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 13 Feb 2025 10:11:37 -0500
Subject: [PATCH 16/16] fixing indexer

---
 client/src/components/FeatureSelection/VectorFeatureChart.vue | 3 +--
 client/src/types.ts                                           | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/client/src/components/FeatureSelection/VectorFeatureChart.vue b/client/src/components/FeatureSelection/VectorFeatureChart.vue
index b1ccc47..9a21f09 100644
--- a/client/src/components/FeatureSelection/VectorFeatureChart.vue
+++ b/client/src/components/FeatureSelection/VectorFeatureChart.vue
@@ -98,8 +98,7 @@ export default defineComponent({
           props.vectorFeatureId,
           props.graphInfo.xAxis,
           props.graphInfo.yAxis,
-          props.graphInfo.filterColumn,
-          props.graphInfo.filterValue,
+          props.graphInfo.indexer,
         );
         graphData.value = data;
         renderGraph(data);
diff --git a/client/src/types.ts b/client/src/types.ts
index 408ba0f..f183345 100644
--- a/client/src/types.ts
+++ b/client/src/types.ts
@@ -771,7 +771,7 @@ export interface VectorFeatureTableGraph {
   type: string;
   xAxis: string;
   yAxis: string;
-  indexer: string;
+  indexer?: string;
 }
 
 export interface FeatureGraphData {