WIP on initializing NE database [#108] (#110)

Replace RegionInfos, CountryInfos and NE index with NaturalEarthDb on init [#108]

* NaturalEarthDb cleans up NE duplicates and is queried in-memory by Places layer
* move ne zoom adjustment and null-handling into layers
* Add QrankDb, but don't use it to rank POIs yet [#29]

Author: bdon

tiles/src/main/java/com/protomaps/basemap/Basemap.java

@@ -3,6 +3,9 @@
 import com.onthegomap.planetiler.ForwardingProfile;
 import com.onthegomap.planetiler.Planetiler;
 import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.util.Downloader;
+import com.protomaps.basemap.feature.NaturalEarthDb;
+import com.protomaps.basemap.feature.QrankDb;
 import com.protomaps.basemap.layers.Boundaries;
 import com.protomaps.basemap.layers.Buildings;
 import com.protomaps.basemap.layers.Earth;
@@ -21,7 +24,7 @@
 
 public class Basemap extends ForwardingProfile {
 
-  public Basemap(Envelope earthWaterBounds) {
+  public Basemap(Envelope earthWaterBounds, NaturalEarthDb naturalEarthDb, QrankDb qrankDb) {
 
     var admin = new Boundaries();
     registerHandler(admin);
@@ -49,12 +52,12 @@ public Basemap(Envelope earthWaterBounds) {
     registerSourceHandler("osm", physicalPoint);
     registerSourceHandler("ne", physicalPoint::processNe);
 
-    var place = new Places();
+    var place = new Places(naturalEarthDb);
     registerHandler(place);
     registerSourceHandler("osm", place);
     registerSourceHandler("ne", place::processNe);
 
-    var poi = new Pois();
+    var poi = new Pois(qrankDb);
     registerHandler(poi);
     registerSourceHandler("osm", poi);
 
@@ -116,20 +119,29 @@ static void run(Arguments args) throws Exception {
     // and a complete high-zoom tileset without having to tile earth/water outside the bbox.
     Envelope earthWaterBounds = args.bounds("osm-earth-water-bounds", "spatial bbox of osm earth+water");
 
+    Path nePath = sourcesDir.resolve("natural_earth_vector.sqlite.zip");
+    String neUrl = "https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip";
+
     String area = args.getString("area", "geofabrik area to download", "monaco");
 
     var planetiler = Planetiler.create(args)
-      .setProfile(new Basemap(earthWaterBounds))
       // (nvkelso 20230817) Order of operations matters here so all NE places can be added to RTree indexes
       //                    before OSM uses them for data joins
-      .addNaturalEarthSource("ne", sourcesDir.resolve("natural_earth_vector.sqlite.zip"),
-        "https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip")
+      .addNaturalEarthSource("ne", nePath, neUrl)
       .addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area)
-      .setOutput(Path.of(area + ".pmtiles"));
-    planetiler.addShapefileSource("osm_water", sourcesDir.resolve("water-polygons-split-3857.zip"),
-      "https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip")
+      .addShapefileSource("osm_water", sourcesDir.resolve("water-polygons-split-3857.zip"),
+        "https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip")
       .addShapefileSource("osm_land", sourcesDir.resolve("land-polygons-split-3857.zip"),
-        "https://osmdata.openstreetmap.de/download/land-polygons-split-3857.zip")
+        "https://osmdata.openstreetmap.de/download/land-polygons-split-3857.zip");
+
+    Downloader.create(planetiler.config(), planetiler.stats()).add("ne", neUrl, nePath)
+      .add("qrank", "https://qrank.wmcloud.org/download/qrank.csv.gz", sourcesDir.resolve("qrank.csv.gz")).run();
+
+    var tmpDir = nePath.resolveSibling(nePath.getFileName() + "-unzipped");
+    var naturalEarthDb = NaturalEarthDb.fromSqlite(nePath, tmpDir);
+    var qrankDb = QrankDb.fromCsv(sourcesDir.resolve("qrank.csv.gz"));
+
+    planetiler.setProfile(new Basemap(earthWaterBounds, naturalEarthDb, qrankDb)).setOutput(Path.of(area + ".pmtiles"))
       .run();
   }
 }