Assign nil geometries to obs with malformed coordinates (#1922) (#1923)

* Assign nil geometries to obs with malformed coordinates

Fixes #1922

* Update test_csv__provider.py

* Update test_csv__provider.py

* Update test_csv__provider.py

---------

Co-authored-by: Tom Kralidis <tomkralidis@gmail.com>

pygeoapi/provider/csv_.py

@@ -143,9 +143,9 @@ def _load(self, offset=0, limit=10, resulttype='results',
                         float(row.pop(self.geometry_y)),
                     ]
                 except ValueError:
-                    msg = f'Skipping row with invalid geometry: {row.get(self.id_field)}'  # noqa
-                    LOGGER.error(msg)
-                    continue
+                    msg = f'Row with invalid geometry: {row.get(self.id_field)}, setting coordinates to None'  # noqa
+                    LOGGER.warning(msg)
+                    coordinates = None
 
                 feature = {'type': 'Feature'}
                 feature['id'] = row.pop(self.id_field)

tests/data/obs_malformatted.csv

@@ -0,0 +1,6 @@
+id,stn_id,datetime,value,lat,long
+371,35,"2001-10-30T14:24:55Z",89.9,45,-75
+377,35,"2002-10-30T18:31:38Z",93.9,45,-75
+238,2147,"2007-10-30T08:57:29Z",103.5,43,-79
+297,2147,"2003-10-30T07:37:29Z",93.5,,
+964,604,"2000-10-30T18:24:39Z",99.9,A,X

tests/test_csv__provider.py

@@ -27,6 +27,8 @@
 #
 # =================================================================
 
+import logging
+
 import pytest
 
 from pygeoapi.provider.base import (ProviderItemNotFoundError,
@@ -35,8 +37,11 @@
 
 from .util import get_test_file_path
 
+LOGGER = logging.getLogger(__name__)
+
 path = get_test_file_path('data/obs.csv')
 stations_path = get_test_file_path('data/station_list.csv')
+malformatted_path = get_test_file_path('data/obs_malformatted.csv')
 
 
 @pytest.fixture()
@@ -67,6 +72,20 @@ def station_config():
     }
 
 
+@pytest.fixture()
+def malformatted_config():
+    return {
+        'name': 'CSV',
+        'type': 'feature',
+        'data': malformatted_path,
+        'id_field': 'id',
+        'geometry': {
+            'x_field': 'long',
+            'y_field': 'lat'
+        }
+    }
+
+
 def test_query(config):
     p = CSVProvider(config)
 
@@ -153,3 +172,17 @@ def test_get_station(station_config):
 
     result = p.get('0-454-2-AWSNAMITAMBO')
     assert result['properties']['station_name'] == 'NAMITAMBO'
+
+
+def test_get_malformed(malformatted_config, caplog):
+    p = CSVProvider(malformatted_config)
+
+    with caplog.at_level(logging.WARNING):
+        results = p.query()
+
+    assert 'Row with invalid geometry' in caplog.text
+    assert len(results['features']) == 5
+    assert results['numberMatched'] == 5
+    assert results['numberReturned'] == 5
+    assert results['features'][3]['geometry']['coordinates'] is None
+    assert results['features'][4]['geometry']['coordinates'] is None