rcat dgo reformat

lib/commons/rscommons/database.py

@@ -243,6 +243,20 @@ def load_igo_attributes(database, fields, where_clause=None):
     return igos
 
 
+def load_dgo_attributes(database, fields, where_clause=None):
+
+    conn = sqlite3.connect(database)
+    conn.row_factory = dict_factory
+    curs = conn.execute('SELECT DGOID, {} FROM vwDgos {}'.format(','.join(fields), 'WHERE {}'.format(where_clause) if where_clause else ''))
+    dgos = {}
+    for row in curs.fetchall():
+        dgos[row['DGOID']] = {}
+        for field in fields:
+            dgos[row['DGOID']][field] = row[field]
+
+    return dgos
+
+
 def write_db_attributes(database, reaches, fields, set_null_first=True, summarize=True):
 
     if len(reaches) < 1:
@@ -295,6 +309,32 @@ def write_db_igo_attributes(database, features, fields, set_null_first=True, sum
         [summarize_reaches(database, field) for field in fields]
 
 
+def write_db_dgo_attributes(database, features, fields, set_null_first=True, summarize=True):
+
+    if len(features) < 1:
+        return
+
+    conn = sqlite3.connect(database)
+    conn.execute('pragma foreign_keys=ON')
+    curs = conn.cursor()
+
+    # Optionally clear all the values in the fields first
+    if set_null_first is True:
+        [curs.execute('UPDATE DGOAttributes SET {} = NULL'.format(field)) for field in fields]
+
+    results = []
+    for reachid, values in features.items():
+        results.append([values[field] if field in values else None for field in fields])
+        results[-1].append(reachid)
+
+    sql = 'UPDATE DGOAttributes SET {} WHERE DGOID = ?'.format(','.join(['{}=?'.format(field) for field in fields]))
+    curs.executemany(sql, results)
+    conn.commit()
+
+    if summarize is True:
+        [summarize_reaches(database, field) for field in fields]
+
+
 def summarize_reaches(database, field):
 
     log = Logger('Database')

packages/rcat/.vscode/launch.json

@@ -26,7 +26,7 @@
             "{env:DATA_ROOT}/rs_context/${input:HUC}/vegetation/historic_veg.tif",
             "{env:DATA_ROOT}/taudem/${input:HUC}/intermediates/pitfill.tif",
             "{env:DATA_ROOT}/anthro/${input:HUC}/outputs/anthro.gpkg/vwIgos",
-            "{env:DATA_ROOT}/anthro/${input:HUC}/inputs/inputs.gpkg/dgo",
+            "{env:DATA_ROOT}/anthro/${input:HUC}/outputs/anthro.gpkg/vwDgos",
             "{env:DATA_ROOT}/anthro/${input:HUC}/outputs/anthro.gpkg/vwReaches",
             "{env:DATA_ROOT}/anthro/${input:HUC}/inputs/inputs.gpkg/roads",
             "{env:DATA_ROOT}/anthro/${input:HUC}/inputs/inputs.gpkg/rails",

packages/rcat/database/data/DGOConv.csv

@@ -0,0 +1 @@
+DGOID,ConvVal,ConvArea,ConvCellCount

packages/rcat/database/data/DGOExRiparian.csv

@@ -0,0 +1 @@
+DGOID,ExRipVal,ExRipArea,ExRipCellCount

packages/rcat/database/data/DGOExVeg.csv

@@ -0,0 +1 @@
+DGOID,ExVegVal,ExVegArea,ExVegCellCount

packages/rcat/database/data/DGOFPAccess.csv

@@ -0,0 +1 @@
+DGOID,AccessVal,CellArea,CellCount

packages/rcat/database/data/DGOHRiparian.csv

@@ -0,0 +1 @@
+DGOID,HRipVal,HRipArea,HRipCellCount

packages/rcat/database/data/DGOHVeg.csv

@@ -0,0 +1 @@
+DGOID,HVegVal,HVegArea,HVegCellCount

packages/rcat/database/data/DGOVegetation.csv

@@ -0,0 +1 @@
+DGOID,VegetationID,Area,CellCount

packages/rcat/database/rcat_schema.sql

@@ -68,50 +68,50 @@ CREATE TABLE VegClassification (
     PRIMARY KEY (Physiognomy, EpochID)
 );
 
-CREATE TABLE IGOVegetation (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOVegetation (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     VegetationID INTEGER REFERENCES VegetationTypes (VegetationID) NOT NULL,
     Area REAL NOT NULL CONSTRAINT CHK_Area CHECK (Area > 0),
     CellCount REAL NOT NULL CONSTRAINT CHK_CellCount CHECK (CellCount > 0)
 );
 
-CREATE TABLE IGOExRiparian (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOExRiparian (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     ExRipVal INTEGER,
     ExRipArea REAL NOT NULL CONSTRAINT CHK_ExRip_Area CHECK (ExRipArea > 0),
     ExRipCellCount REAL NOT NULL CONSTRAINT CHK_ExRip_CellCount CHECK (ExRipCellCount > 0)
 );
 
-CREATE TABLE IGOHRiparian (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOHRiparian (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     HRipVal INTEGER,
     HRipArea REAL NOT NULL CONSTRAINT CHK_HRip_Area CHECK (HRipArea > 0),
     HRipCellCount REAL NOT NULL CONSTRAINT CHK_HRip_CellCount CHECK (HRipCellCount > 0)
 );
 
-CREATE TABLE IGOExVeg (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOExVeg (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     ExVegVal INTEGER,
     ExVegArea REAL NOT NULL CONSTRAINT CHK_ExVeg_Area CHECK (ExVegArea > 0),
     ExVegCellCount REAL NOT NULL CONSTRAINT CHK_ExVeg_CellCount CHECK (ExVegCellCount > 0)
 );
 
-CREATE TABLE IGOHVeg (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOHVeg (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     HVegVal INTEGER,
     HVegArea REAL NOT NULL CONSTRAINT CHK_HVeg_Area CHECK (HVegArea > 0),
     HVegCellCount REAL NOT NULL CONSTRAINT CHK_HVeg_CellCount CHECK (HVegCellCount > 0)
 );
 
-CREATE TABLE IGOConv (
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOConv (
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     ConvVal INTEGER,
     ConvArea REAL NOT NULL CONSTRAINT CHK_Conv_Area CHECK (ConvArea > 0),
     ConvCellCount REAL NOT NULL CONSTRAINT CHK_Conv_CellCount CHECK (ConvCellCount > 0)
 );
 
-CREATE TABLE IGOFPAccess(
-    IGOID INTEGER REFERENCES IGOAttributes ON DELETE CASCADE NOT NULL,
+CREATE TABLE DGOFPAccess(
+    DGOID INTEGER REFERENCES DGOAttributes ON DELETE CASCADE NOT NULL,
     AccessVal INTEGER,
     CellArea REAL,
     CellCount INTEGER
@@ -171,8 +171,6 @@ CREATE TABLE IGOAttributes (
     LevelPathI REAL,
     seg_distance REAL,
     stream_size INTEGER,
-    window_area REAL,
-    window_length REAL,
     LUI REAL,
     FloodplainAccess REAL,
     FromConifer REAL,
@@ -196,6 +194,32 @@ CREATE TABLE IGOAttributes (
 
 );
 
+CREATE TABLE DGOAttributes (
+    DGOID INTEGER PRIMARY KEY NOT NULL,
+    LevelPathI REAL,
+    seg_distance REAL,
+    centerline_length REAL,
+    segment_area REAL,
+    LUI REAL,
+    FloodplainAccess REAL,
+    FromConifer REAL,
+    FromDevegetated REAL,
+    FromGrassShrubland REAL,
+    NoChange REAL,
+    GrassShrubland REAL,
+    Devegetation REAL,
+    Conifer REAL,
+    Invasive REAL,
+    Development REAL,
+    Agriculture REAL,
+    NonRiparian REAL,
+    ExistingRiparianMean REAL,
+    HistoricRiparianMean REAL,
+    RiparianDeparture REAL,
+    Condition REAL
+
+);
+
 CREATE TABLE ReachAttributes (
     ReachID INTEGER PRIMARY KEY NOT NULL,
     ReachCode INTEGER,
@@ -245,6 +269,10 @@ FROM IGOAttributes I
     INNER JOIN Conversions C ON I.ConversionID = C.ConversionID AND I.LevelID = C.LevelID
     INNER JOIN Departure D ON I.RiparianDepartureID = D.RiparianDepartureID;
 
+CREATE VIEW vwDgos AS SELECT DA.*, G.geom
+FROM DGOAttributes DA
+    INNER JOIN DGOGeometry G ON DA.DGOID = G.DGOID;
+
 
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('Watersheds', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('Epochs', 'attributes');
@@ -254,13 +282,13 @@ INSERT INTO gpkg_contents (table_name, data_type) VALUES ('Conversions', 'attrib
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('Departure', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('VegetationTypes', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('VegClassification', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOVegetation', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOExRiparian', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOHRiparian', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOExVeg', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOHVeg', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOConv', 'attributes');
-INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOFPAccess', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOVegetation', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOExRiparian', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOHRiparian', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOExVeg', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOHVeg', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOConv', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOFPAccess', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachVegetation', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachExRiparian', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachHRiparian', 'attributes');
@@ -269,4 +297,5 @@ INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachHVeg', 'attribut
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachConv', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachFPAccess', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('IGOAttributes', 'attributes');
+INSERT INTO gpkg_contents (table_name, data_type) VALUES ('DGOAttributes', 'attributes');
 INSERT INTO gpkg_contents (table_name, data_type) VALUES ('ReachAttributes', 'attributes')

packages/rcat/rcat/lib/igo_vegetation.py

@@ -16,12 +16,13 @@
 from rscommons.database import SQLiteCon
 
 
-def igo_vegetation(windows: dict, raster: str, out_gpkg_path: str):  # , large_rivers: dict):
+def igo_vegetation(raster: str, dgo: dict, out_gpkg_path: str):  # , large_rivers: dict):
     """Summarizes vegetation raster datasets onto IGOs based on moving windows
 
     Arguments:
         windows (dict): dictionary with moving window features associated with each IGO
         raster (str): Path to raster dataset to summarize
+        dgo (dict): A dictionary where key = DGOID and val = shapely geometry of dgo with large rivers cut out
         out_gpkg_path: Path to the geopackage containing the tables to fill out
     """
 
@@ -37,17 +38,17 @@ def igo_vegetation(windows: dict, raster: str, out_gpkg_path: str):  # , large_r
     with rasterio.open(raster) as src:
 
         veg_counts = []
-        for igoid, window in windows.items():
+        for dgoid, dgo_geom in dgo.items():
             try:
-                raw_raster = mask(src, [window], crop=True)[0]
+                raw_raster = mask(src, [dgo_geom], crop=True)[0]
                 mask_raster = np.ma.masked_values(raw_raster, src.nodata)
 
                 for oldvalue in np.unique(mask_raster):
                     if oldvalue is not np.ma.masked:
                         cell_count = np.count_nonzero(mask_raster == oldvalue)
-                        veg_counts.append([igoid, int(oldvalue), cell_count * cell_area, cell_count])
+                        veg_counts.append([dgoid, int(oldvalue), cell_count * cell_area, cell_count])
             except Exception as ex:
-                log.warning(f'Error obtaining land cover raster values for igo ID {igoid}')
+                log.warning(f'Error obtaining land cover raster values for dgo ID {dgoid}')
                 log.warning(ex)
 
     with SQLiteCon(out_gpkg_path) as database:
@@ -58,22 +59,22 @@ def igo_vegetation(windows: dict, raster: str, out_gpkg_path: str):  # , large_r
             if int(veg_record[1]) != -9999:
                 try:
                     if os.path.basename(raster) in ['existing_veg.tif', 'historic_veg.tif']:
-                        database.conn.execute('INSERT INTO IGOVegetation (IGOID, VegetationID, Area, CellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOVegetation (DGOID, VegetationID, Area, CellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'ex_riparian.tif':
-                        database.conn.execute('INSERT INTO IGOExRiparian (IGOID, ExRipVal, ExRipArea, ExRipCellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOExRiparian (DGOID, ExRipVal, ExRipArea, ExRipCellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'hist_riparian.tif':
-                        database.conn.execute('INSERT INTO IGOHRiparian (IGOID, HRipVal, HRipArea, HRipCellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOHRiparian (DGOID, HRipVal, HRipArea, HRipCellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'ex_vegetated.tif':
-                        database.conn.execute('INSERT INTO IGOExVeg (IGOID, ExVegVal, ExVegArea, ExVegCellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOExVeg (DGOID, ExVegVal, ExVegArea, ExVegCellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'hist_vegetated.tif':
-                        database.conn.execute('INSERT INTO IGOHVeg (IGOID, HVegVal, HVegArea, HVegCellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOHVeg (DGOID, HVegVal, HVegArea, HVegCellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'conversion.tif':
-                        database.conn.execute('INSERT INTO IGOConv (IGOID, ConvVal, ConvArea, ConvCellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOConv (DGOID, ConvVal, ConvArea, ConvCellCount) VALUES (?, ?, ?, ?)', veg_record)
                     elif os.path.basename(raster) == 'fp_access.tif':
-                        database.conn.execute('INSERT INTO IGOFPAccess (IGOID, AccessVal, CellArea, CellCount) VALUES (?, ?, ?, ?)', veg_record)
+                        database.conn.execute('INSERT INTO DGOFPAccess (DGOID, AccessVal, CellArea, CellCount) VALUES (?, ?, ?, ?)', veg_record)
                 except sqlite3.IntegrityError as err:
                     # THis is likely a constraint error.
-                    errstr = "Integrity Error when inserting records: IGOID: {} VegetationID: {}".format(veg_record[0], veg_record[1])
+                    errstr = "Integrity Error when inserting records: DGOID: {} VegetationID: {}".format(veg_record[0], veg_record[1])
                     log.error(errstr)
                     errs += 1
                 except sqlite3.Error as err: