Add dash dashboard

.gitignore

@@ -135,4 +135,8 @@ dmypy.json
 .pytype/
 
 # Cython debug symbols
-cython_debug/
+cython_debug/
+data/
+
+.vscode/
+dashboard-cache/

README.md

@@ -1,21 +1,22 @@
-# lidar-postgis-anal
+# PostGIS with Dash to query and show lidar data
 
 ## Description
 
+[Data](https://skfb.ly/6QUwN)
 
 ## Requirements
 
 1. Python 3.8 or higher.
+2. Anaconda or Miniconda
+3. [docker compose v2](https://github.com/docker/compose)
 
 ## How to run
 
 Install dependencies:
 ```
-pip install -r. /requirements.txt
-```
-
-For development:
-```
-pip install -r ./requirements.txt -r ./requirements.dev.txt
+conda create -n env_name python=3.9 pip -y
+conda activate env_name
+conda update --file ./environment.yaml
 ```
 
+Use [Mamba instead conda](https://github.com/mamba-org/mamba) to speedup dependency resolving.

configs/db/local.yaml

@@ -0,0 +1,6 @@
+user: test
+pass: 267
+db_name: geo-db
+host: localhost
+port: 5432
+url: postgresql://${.user}:${.pass}@${.host}:${.port}/${.db_name}

configs/insert_data.yaml

@@ -0,0 +1,20 @@
+hydra:
+  run:
+    dir: .
+  output_subdir: null
+
+defaults:
+  - db: local
+  - _self_
+
+data:
+  filepath: ./data/Carola_PointCloud.ply
+
+loader:
+ _target_: loader.ply_loader.PLYLoader
+
+data_splitting:
+  voxel_size: 2.5
+  chunk_size: 2_000_000
+
+

dashboard.py

@@ -0,0 +1,155 @@
+from operator import attrgetter
+import numbers
+
+from sqlalchemy import create_engine
+from sqlalchemy import sql
+from sqlalchemy.orm import sessionmaker
+from geoalchemy2 import func as geosql
+import geopandas as geo
+import numpy as np
+import dash_vtk
+from dash import Dash, html, dcc, Input, Output, State
+from dash.long_callback import DiskcacheLongCallbackManager
+import diskcache
+import hydra
+
+from db import LazPoints
+
+with hydra.initialize("configs") as config_dir:
+    CONFIG = hydra.compose("insert_data")
+
+FILE_DROPDOWN = "file-selection-id"
+CHUNK_DROP_DOWN = "chunk-dropdown-id"
+SIGN_INFO_TABLE_ID = "sign-info-table"
+PLOT_3D_ID = "3d-scatter-id"
+EXTERNAL_DROPDOWN = "external-id-field"
+PROGRESS_ID = "progress-id"
+RADIUS_SLIDER_ID = "selection-radius-range"
+PROGRESS_BAR_ID = "progress-bar-id"
+BUTTON_ID = "draw-button-id"
+
+CACHE_DIR = "./dashboard-cache"
+
+cache = diskcache.Cache(CACHE_DIR)
+long_callback_manager = DiskcacheLongCallbackManager(cache)
+
+app = Dash(__name__)
+
+
+app.layout = html.Div([
+    html.Div(children=[dcc.Link("View control", href="https://dash.plotly.com/vtk/intro#view")]),
+    html.Label("File:", id="start"),
+    dcc.Dropdown(id=FILE_DROPDOWN),
+    html.Label("Chunk id:"),
+    dcc.Dropdown(id=CHUNK_DROP_DOWN, multi=True),
+    html.Div(children=[html.Button("Query points and draw", id=BUTTON_ID, style={"width": "25%"})]),
+    html.Progress(id=PROGRESS_BAR_ID, max=str(100),
+                  title="Loading...", style={"visibility": "hidden"}),
+    html.Div(id=PLOT_3D_ID, style={"height": "75vh"})
+]
+)
+
+
+@ app.callback(
+    Output(FILE_DROPDOWN, "value"),
+    Output(FILE_DROPDOWN, "options"),
+    Input("start", "style")
+)
+def select_files(_):
+    engine = create_engine(CONFIG.db.url)
+    Session = sessionmaker(engine)
+
+    with Session.begin() as session:
+        all_files = list(map(attrgetter("file"), session.query(LazPoints.file).distinct()))
+
+    engine.dispose()
+
+    return all_files[0], all_files
+
+
+@ app.callback(
+    Output(CHUNK_DROP_DOWN, "value"),
+    Output(CHUNK_DROP_DOWN, "options"),
+    Input(FILE_DROPDOWN, "value"),
+)
+def select_chunk_ids(file):
+    engine = create_engine(CONFIG.db.url)
+    Session = sessionmaker(engine)
+
+    with Session.begin() as session:
+        all_chunk_ids = list(map(attrgetter("chunk_id"), session.query(
+            LazPoints.chunk_id).where(LazPoints.file == file)))
+
+    engine.dispose()
+
+    return all_chunk_ids[0], all_chunk_ids
+
+
+@ app.long_callback(
+    Output(PLOT_3D_ID, "children"),
+    State(FILE_DROPDOWN, "value"),
+    State(CHUNK_DROP_DOWN, "value"),
+    Input(BUTTON_ID, "n_clicks"),
+    manager=long_callback_manager,
+    running=[
+        (Output(FILE_DROPDOWN, "disabled"), True, False),
+        (Output(CHUNK_DROP_DOWN, "disabled"), True, False),
+        (Output(BUTTON_ID, "disabled"), True, False),
+        (
+            Output(PROGRESS_BAR_ID, "style"),
+            {"visibility": "visible"},
+            {"visibility": "hidden"},
+        )
+    ],
+    progress=[Output(PROGRESS_BAR_ID, "value")],
+    prevent_initial_call=True
+)
+def select_chunk(set_progress, file_path, chunk_ids, n_click):
+    if isinstance(chunk_ids, numbers.Number):
+        chunk_ids = [chunk_ids]
+
+    if not chunk_ids:
+        return dash_vtk.View(
+            background=[0, 0, 0]
+        )
+
+    set_progress([str(10)])
+
+    engine = create_engine(CONFIG.db.url)
+    Session = sessionmaker(engine)
+
+    geom_col_name = "geom"
+
+    with Session.begin() as session:
+        query_points = sql.select(geosql.ST_DumpPoints(LazPoints.points).geom.label(
+            geom_col_name)) \
+            .filter(LazPoints.file == file_path) \
+            .filter(LazPoints.chunk_id.in_(chunk_ids))
+
+        points = geo.read_postgis(
+            query_points, session.connection(), geom_col=geom_col_name)
+
+    engine.dispose()
+
+    set_progress([str(50)])
+
+    coords = np.vstack(
+        (points[geom_col_name].x, points[geom_col_name].y, points[geom_col_name].z)).T.reshape(-1)
+
+    vtk_view = dash_vtk.View(
+        [
+            dash_vtk.PointCloudRepresentation(
+                xyz=coords,
+                property={"pointSize": 2}
+            )
+        ],
+        background=[0, 0, 0]
+    )
+
+    set_progress([str(100)])
+
+    return vtk_view
+
+
+if __name__ == "__main__":
+    app.run_server(debug=True, dev_tools_ui=True)

db/__init__.py

@@ -0,0 +1 @@
+from .laz_points import LazPoints

db/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()

db/laz_points.py

@@ -0,0 +1,14 @@
+from sqlalchemy import Column, BigInteger, String, Integer
+from geoalchemy2 import Geometry
+
+from .base import Base
+
+
+class LazPoints(Base):
+    __tablename__ = "LidarPointsPly"
+
+    id = Column(BigInteger, primary_key=True, autoincrement=True)
+    chunk_id = Column(Integer, nullable=False)
+    file = Column(String, nullable=False)
+    points = Column(Geometry("MULTIPOINTZ", dimension=3,
+                    spatial_index=False, nullable=False, use_N_D_index=False))

environment.yaml

@@ -0,0 +1,27 @@
+channels:
+  - defaults
+dependencies:
+  - python=3.9
+  - pip
+  - numpy
+  - conda-forge::psycopg2==2.9
+  - tqdm==4.63
+  - ipython
+  - autopep8
+  - plotly::plotly==5.7.0
+  - ipykernel
+  - conda-forge::sqlalchemy==1.4
+  - conda-forge::geopandas==0.10
+  - h5py=3.6
+  - conda-forge::shapely==1.8
+  - pip:
+    - geoalchemy2~=0.11.0
+    - sqlalchemy-utils~=0.38.0
+    - diskcache~=5.4.0
+    - psutil~=5.9.0
+    - dash[diskcache]~=2.4.0
+    - dash-bootstrap-components~=1.1.0
+    - plyfile~=0.7.4
+    - hydra-core~=1.1.1
+    - dash-vtk~=0.0.9
+    - vtk~=9.1.0

insert_data.py

@@ -0,0 +1,113 @@
+import tempfile
+import pathlib
+import os
+
+import numpy as np
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+from geoalchemy2.shape import from_shape
+from shapely.geometry import MultiPoint
+import h5py
+import hydra
+import numpy as np
+from tqdm import tqdm
+from hydra.utils import instantiate
+
+from db import LazPoints
+from db.base import Base
+
+from loader import BasePointCloudLoader
+from utils import append_points, DATASET_PATH
+
+
+def insert_points(session: Session, file_path: str, file: h5py.File, chunk_id: int):
+    res = session.query(LazPoints).filter(LazPoints.chunk_id == chunk_id).filter(
+        LazPoints.file == file_path).one_or_none()
+
+    if res is not None:
+        return
+
+    points = file.get(DATASET_PATH)[:]
+
+    multi_point = MultiPoint(points)
+    session.add(LazPoints(file=file_path, chunk_id=chunk_id, points=from_shape(multi_point)))
+
+
+def generate_bounds(min_value: float, max_value: float, step: float):
+    values = []
+    value = min_value
+
+    while value < max_value:
+        values.append(value)
+        value += step
+
+    values.append(max_value)
+
+    return np.array(values)
+
+
+def get_chunk_indices(points: np.ndarray, x_bounds: np.ndarray, y_bounds: np.ndarray, z_bounds: np.ndarray):
+    x_indices = np.searchsorted(x_bounds, points[:, 0], side="right") - 1  # 0 to n - 1
+    y_indices = np.searchsorted(y_bounds, points[:, 1], side="right") - 1
+    z_indices = np.searchsorted(z_bounds, points[:, 2], side="right") - 1
+
+    # Convert three dimensional index to the one dimensional
+    width = len(x_bounds) - 1
+    height = len(y_bounds) - 1
+
+    return z_indices * width * height + y_indices * width + x_indices
+
+
+def insert_data(session_factory, path_to_file: str, loader_config, chunk_size: int, voxel_size: float):
+    loader: BasePointCloudLoader = instantiate(loader_config, path_to_file)
+
+    file_loc = pathlib.Path(path_to_file).as_posix()
+
+    min_bounds, max_bounds = loader.get_bounds()
+
+    x_intervals = generate_bounds(min_bounds[0], max_bounds[0], voxel_size)
+    y_intervals = generate_bounds(min_bounds[1], max_bounds[1], voxel_size)
+    z_intervals = generate_bounds(min_bounds[2], max_bounds[2], voxel_size)
+
+    files = []
+    chunk_ids = []
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        for chunk_xyz in loader.iter_chunks(chunk_size):
+            chunk_index_per_point = get_chunk_indices(
+                chunk_xyz, x_intervals, y_intervals, z_intervals)
+
+            for chunk_index in tqdm(set(chunk_index_per_point), desc="Save to hdf"):
+                file_path = os.path.join(tmp_dir, f"chunk_{chunk_index}.h5")
+                files.append(file_path)
+                chunk_ids.append(int(chunk_index))
+
+                with h5py.File(file_path, "a") as hdf_file:
+                    local_chunk = chunk_xyz[chunk_index_per_point == chunk_index]
+                    append_points(hdf_file, local_chunk)
+
+                del file_path
+
+        with session_factory() as session:
+            for file, chunk_id in tqdm(zip(files, chunk_ids), total=len(files), desc="Insert chunks"):
+                with h5py.File(file, "r") as hdf_file:
+                    insert_points(session, file_loc, hdf_file, chunk_id)
+                session.commit()
+
+
+@hydra.main(config_path="configs", config_name="insert_data")
+def main(config):
+
+    engine = create_engine(config.db.url)
+
+    try:
+        Base.metadata.create_all(engine)
+        CustomSession = sessionmaker(engine)
+        insert_data(CustomSession, config.data.filepath, config.loader,
+                    config.data_splitting.chunk_size, config.data_splitting.voxel_size)
+    finally:
+        engine.dispose()
+
+
+if __name__ == "__main__":
+    main()

loader/__init__.py

@@ -0,0 +1,2 @@
+from .base_loader import BasePointCloudLoader
+from .ply_loader import PLYLoader