Merge pull request #47 from lonvia/package-analyser

Allow building packages and wheels

.github/workflows/ci-build-tests.yml

@@ -12,8 +12,8 @@ jobs:
     - name: Install python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
-        pip install -r requirements.txt
+        pip install flake8 pytest pybind11
+        pip install .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -28,4 +28,5 @@ jobs:
         sudo -u postgres createuser -s runner
     - name: Test with pytest
       run: |
-        pytest tests/
+        python3 setup.py build
+        pytest tests

.gitignore

@@ -1,7 +1,5 @@
 *.pyc
+src/nominatim_data_analyser.egg-info
+dist
 
-analyser/config/config.yaml
-
-.mason
-mason_packages
-build
+src/nominatim_data_analyser/config/config.yaml

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "clustering-vt/.mason"]
-	path = clustering-vt/.mason
-	url = https://github.com/mapbox/mason

MANIFEST.in

@@ -0,0 +1,3 @@
+include src/nominatim_data_analyser/rules_specifications/*.yaml
+include src/nominatim_data_analyser/config/default.yaml
+recursive-include contrib *

README.md

@@ -9,46 +9,24 @@ The repository containing the frontend of the tool can be found [there](https://
 
 # Installation procedure
 
-## Dependencies
-
-### Code
-
 Clone this repository by running:
 
 ```
-git clone --recursive https://github.com/osm-search/Nominatim-Data-Analyser
+git clone https://github.com/osm-search/Nominatim-Data-Analyser
 ```
 
-### Python
-
-Install python (version 3.5+) and pip.
+Then you can compile everything with
 
-Install the following python packages:
+    python3 setup.py build
 
-```
-pip install pyyaml geojson wheel psycopg2
-```
+Or you can directly compile and install the analyser with
 
-### Clustering-vt
-
-To set up clustering-vt g++17 minimum is needed:
-
-```
-cd clustering-vt/
-make
-```
+    pip install .
 
 ## Database
 
-Make sure to have a Nominatim database set up on your server.
-
-The following index has to be created in order for the tool to work:
-
-```
-CREATE INDEX IF NOT EXISTS planet_osm_rels_parts_idx ON planet_osm_rels USING gin(parts);
-```
-
-You need to run the tool with the nominatim user and/or you should check the database connection settings in ```analyser/database/connection.py```.
+Make sure to have a Nominatim database set up on your server. You can change
+the database DSN by supplying a custom config.yaml file.
 
 ## Configuration file
 
@@ -61,12 +39,16 @@ To set up the frontend, please check the frontend [repository](https://github.co
 For the webapp to fetch the data extracted by the analyser, you need to serve the ```<RulesFolderPath>``` defined in ```analyser/config/config.yaml``` of the QA Data Analyser Tool with a web server. It should be accessible through the ```<WebPrefixPath>``` also defined in the configuration of the QA Data Analyser Tool.
 
 # Running the analyser
-
-You can run the tool's analyser with the integrated cli.py:
+
+Analysis is run with the nominatim-data-analyser tool:
+
 * --execute-all: Executes all QA rules.
 * --filter [rules_names…]: Filters some QA rules so they are not executed.
 * --execute-one <rule_name>: Executes the given QA rule.
 
+During development you can run the same tool directly from the source tree
+after having built everything using the supplied `cli.py`.
+
 # Tests
 
 [Pytest](https://docs.pytest.org/en/6.2.x/getting-started.html) is used for the tests and it should be installed:
@@ -75,7 +57,7 @@ You can run the tool's analyser with the integrated cli.py:
 pip install pytest
 ```
 
-To run the tests for the analyser: execute ```pytest tests/analyser``` command at the root folder of the project.
+To run the tests for the analyser: execute ```pytest``` command at the root folder of the project.
 
 # Reporting errors in the rule
 

cli.py

@@ -1,20 +1,19 @@
-from analyser.core.core import Core
-import argparse
+#!/usr/bin/python3
+import sys
+import sysconfig
+from pathlib import Path
 
-parser = argparse.ArgumentParser(prog='nominatim-analyser')
+SRC_DIR = Path(__file__, '..').resolve()
 
-parser.add_argument('--execute-all', action='store_true', help='Executes all the QA rules')
-parser.add_argument('--filter', metavar='<QA rules names>', nargs='+', help='Filters some QA rules so they are not executed.')
-parser.add_argument('--execute-one', metavar='<QA rule name>', action='store', type=str, help='Executes the given QA rule')
+BUILD_DIR = f"build/lib.{sysconfig.get_platform()}-{sys.version_info[0]}.{sys.version_info[1]}"
 
-args = parser.parse_args()
+if not (SRC_DIR / BUILD_DIR).exists():
+    BUILD_DIR = f"build/lib.{sysconfig.get_platform()}-{sys.implementation.cache_tag}"
 
-#Executes all the QA rules. If a filter is given, these rules are excluded from the execution.
-if args.execute_all:
-    if args.filter:
-        Core().execute_all(args.filter)
-    else:
-        Core().execute_all()
-elif args.execute_one:
-    #Execute the given QA rule.
-    Core().execute_one(args.execute_one)
+if (SRC_DIR / BUILD_DIR).exists():
+    sys.path.insert(0, str(SRC_DIR / BUILD_DIR))
+
+
+from nominatim_data_analyser.cli import cli
+
+sys.exit(cli())

clustering-vt/clustering-vt.cpp

@@ -1,11 +1,11 @@
 
 #include <mapbox/geojson.hpp>
 #include <mapbox/geojson_impl.hpp>
-#include <rapidjson/document.h>
-#include "rapidjson/istreamwrapper.h"
 #include <vtzero/builder.hpp>
 #include <vtzero/index.hpp>
 
+#include <pybind11/pybind11.h>
+
 #define DEBUG_TIMER false
 
 #include <supercluster.hpp>
@@ -211,55 +211,36 @@ void generate_tiles(const short zoom,
     }
 }
 
-int main(int argc, char *argv[]) {
-    if (argc == 1) {
-        std::cerr << "The output folder is missing!\n";
-        return 1;
-    }
-
-    if (argc == 2) {
-        std::cerr << "The radius is missing!\n";
-        return 1;
-    }
-
-    //Read GeoJSON data from stdin
-    std::cin.sync_with_stdio(false);
-    if (std::cin.rdbuf()->in_avail()) {
-
-        mapbox::supercluster::Timer timer;
+int cluster(std::string const outdir, int radius, pybind11::iterable const json_features) {
+    mapbox::supercluster::Timer timer;
 
-        rapidjson::IStreamWrapper isw(std::cin);
-        mapbox::geojson::rapidjson_document d;
-        d.ParseStream<rapidjson::IStreamWrapper>(isw);
+    mapbox::feature::feature_collection<double> features;
 
-        timer("parse JSON");
+    for (auto const &obj: json_features) {
+        std::string feat = obj.cast<std::string>();
+        mapbox::geojson::feature feature = mapbox::geojson::parse<mapbox::geojson::feature>(feat);
+        features.push_back(feature);
+    }
 
-        const auto &json_features = d["features"];
+    timer("load features");
 
-        mapbox::feature::feature_collection<double> features;
-        features.reserve(json_features.Size());
+    mapbox::supercluster::Options options;
+    options.maxZoom = maxZoom;
+    options.radius = radius;
+    options.extent = 256;
+    mapbox::supercluster::Supercluster index(features, options);
 
-        for (auto itr = json_features.Begin(); itr != json_features.End(); ++itr) {
-            mapbox::feature::feature<double> feature = mapbox::geojson::convert<mapbox::feature::feature<double>>(*itr);
-            features.push_back(feature);
-        }
-        timer("convert to geometry.hpp");
+    timer("total supercluster time");
 
-        mapbox::supercluster::Options options;
-        options.maxZoom = maxZoom;
-        options.radius = std::atoi(argv[2]);
-        options.extent = 256;
-        mapbox::supercluster::Supercluster index(features, options);
+    generate_tiles(0, 0, 0, outdir, index);
 
-        timer("total supercluster time");
+    timer("total tiles generation time");
+    std::cout << "Tiles created: " << amountCreated << "\n";
+    std::cout << "Tiles removed: " << amountRemoved << "\n";
 
-        generate_tiles(0, 0, 0, argv[1], index);
+    return 0;
+}
 
-        timer("total tiles generation time");
-        std::cout << "Tiles created: " << amountCreated << "\n";
-        std::cout << "Tiles removed: " << amountRemoved << "\n";
-    }else {
-        std::cerr << "GeoJSON is needed in stdin!\n";
-        return 1;
-    }
-}
+PYBIND11_MODULE(clustering_vt, m) {
+    m.def("cluster", &cluster);
+}

contrib/geojson/0.4.3/include/mapbox/geojson.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <mapbox/geometry.hpp>
+#include <mapbox/feature.hpp>
+#include <mapbox/variant.hpp>
+
+namespace mapbox {
+namespace geojson {
+
+using empty               = mapbox::geometry::empty;
+using point               = mapbox::geometry::point<double>;
+using multi_point         = mapbox::geometry::multi_point<double>;
+using line_string         = mapbox::geometry::line_string<double>;
+using linear_ring         = mapbox::geometry::linear_ring<double>;
+using multi_line_string   = mapbox::geometry::multi_line_string<double>;
+using polygon             = mapbox::geometry::polygon<double>;
+using multi_polygon       = mapbox::geometry::multi_polygon<double>;
+using geometry            = mapbox::geometry::geometry<double>;
+using geometry_collection = mapbox::geometry::geometry_collection<double>;
+
+using value               = mapbox::feature::value;
+using null_value_t        = mapbox::feature::null_value_t;
+using identifier          = mapbox::feature::identifier;
+using feature             = mapbox::feature::feature<double>;
+using feature_collection  = mapbox::feature::feature_collection<double>;
+
+// Parse inputs of known types. Instantiations are provided for geometry, feature, and
+// feature_collection.
+template <class T>
+T parse(const std::string &);
+
+// Parse any GeoJSON type.
+using geojson = mapbox::util::variant<geometry, feature, feature_collection>;
+geojson parse(const std::string &);
+
+// Stringify inputs of known types. Instantiations are provided for geometry, feature, and
+// feature_collection.
+template <class T>
+std::string stringify(const T &);
+
+// Stringify any GeoJSON type.
+std::string stringify(const geojson &);
+
+} // namespace geojson
+} // namespace mapbox

contrib/geojson/0.4.3/include/mapbox/geojson/rapidjson.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <rapidjson/document.h>
+#include <mapbox/geojson.hpp>
+
+namespace mapbox {
+namespace geojson {
+
+// Use the CrtAllocator, because the MemoryPoolAllocator is broken on ARM
+// https://github.com/miloyip/rapidjson/issues/200, 301, 388
+using rapidjson_allocator = rapidjson::CrtAllocator;
+using rapidjson_document = rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson_allocator>;
+using rapidjson_value = rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson_allocator>;
+
+// Convert inputs of known types. Instantiations are provided for geometry, feature, and
+// feature_collection.
+template <typename T>
+T convert(const rapidjson_value &);
+
+// Convert any GeoJSON type.
+geojson convert(const rapidjson_value &);
+
+// Convert back to rapidjson value. Instantiations are provided for geometry, feature, and
+// feature_collection.
+template <typename T>
+rapidjson_value convert(const T &, rapidjson_allocator&);
+
+// Convert any GeoJSON type.
+rapidjson_value convert(const geojson &, rapidjson_allocator&);
+
+} // namespace geojson
+} // namespace mapbox