qrank db wip [#29]

Author: bdon

tiles/src/main/java/com/protomaps/basemap/Basemap.java

@@ -5,6 +5,7 @@
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.util.Downloader;
 import com.protomaps.basemap.feature.NaturalEarthDb;
+import com.protomaps.basemap.feature.QrankDb;
 import com.protomaps.basemap.layers.Boundaries;
 import com.protomaps.basemap.layers.Buildings;
 import com.protomaps.basemap.layers.Earth;
@@ -23,7 +24,7 @@
 
 public class Basemap extends ForwardingProfile {
 
-  public Basemap(Envelope earthWaterBounds, NaturalEarthDb naturalEarth) {
+  public Basemap(Envelope earthWaterBounds, NaturalEarthDb naturalEarthDb, QrankDb qrankDb) {
 
     var admin = new Boundaries();
     registerHandler(admin);
@@ -51,12 +52,12 @@ public Basemap(Envelope earthWaterBounds, NaturalEarthDb naturalEarth) {
     registerSourceHandler("osm", physicalPoint);
     registerSourceHandler("ne", physicalPoint::processNe);
 
-    var place = new Places();
+    var place = new Places(naturalEarthDb);
     registerHandler(place);
     registerSourceHandler("osm", place);
     registerSourceHandler("ne", place::processNe);
 
-    var poi = new Pois();
+    var poi = new Pois(qrankDb);
     registerHandler(poi);
     registerSourceHandler("osm", poi);
 
@@ -133,12 +134,14 @@ static void run(Arguments args) throws Exception {
       .addShapefileSource("osm_land", sourcesDir.resolve("land-polygons-split-3857.zip"),
         "https://osmdata.openstreetmap.de/download/land-polygons-split-3857.zip");
 
-    Downloader.create(planetiler.config(), planetiler.stats()).add("ne", neUrl, nePath).run();
+    Downloader.create(planetiler.config(), planetiler.stats()).add("ne", neUrl, nePath)
+      .add("qrank", "https://qrank.wmcloud.org/download/qrank.csv.gz", sourcesDir.resolve("qrank.csv.gz")).run();
 
     var tmpDir = nePath.resolveSibling(nePath.getFileName() + "-unzipped");
     var naturalEarthDb = NaturalEarthDb.fromSqlite(nePath, tmpDir);
-    System.out.println(naturalEarthDb.getPopulatedPlaceByWikidataId("Q60"));
+    var qrankDb = QrankDb.fromCsv(sourcesDir.resolve("qrank.csv.gz"));
 
-    planetiler.setProfile(new Basemap(earthWaterBounds, naturalEarthDb)).setOutput(Path.of(area + ".pmtiles")).run();
+    planetiler.setProfile(new Basemap(earthWaterBounds, naturalEarthDb, qrankDb)).setOutput(Path.of(area + ".pmtiles"))
+      .run();
   }
 }

tiles/src/main/java/com/protomaps/basemap/feature/QrankDb.java

@@ -1,2 +1,40 @@
-package com.protomaps.basemap.feature;public class QrankDb {
+package com.protomaps.basemap.feature;
+
+import com.carrotsearch.hppc.LongLongHashMap;
+import java.io.*;
+import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+
+public class QrankDb {
+
+  private LongLongHashMap db;
+
+  public QrankDb(LongLongHashMap db) {
+    this.db = db;
+  }
+
+  public long get(long wikidataId) {
+    return this.db.get(wikidataId);
+  }
+
+  public static QrankDb fromCsv(Path csvPath) throws IOException {
+    GZIPInputStream gzip = new GZIPInputStream(new FileInputStream(csvPath.toFile()));
+    BufferedReader br = new BufferedReader(new InputStreamReader(gzip));
+
+    String content;
+    long startTime = System.nanoTime();
+
+    LongLongHashMap db = new LongLongHashMap();
+    br.readLine(); // header
+    while ((content = br.readLine()) != null) {
+      var split = content.split(",");
+      long id = Long.parseLong(split[0].substring(1));
+      long rank = Long.parseLong(split[1]);
+      db.put(id, rank);
+    }
+    long endTime = System.nanoTime();
+    long elapsedTimeMillis = (endTime - startTime) / 1_000_000;
+    System.out.println(elapsedTimeMillis);
+    return new QrankDb(db);
+  }
 }

tiles/src/main/java/com/protomaps/basemap/layers/Places.java

@@ -13,6 +13,7 @@
 import com.onthegomap.planetiler.util.ZoomFunction;
 import com.protomaps.basemap.feature.CountryInfos;
 import com.protomaps.basemap.feature.FeatureId;
+import com.protomaps.basemap.feature.NaturalEarthDb;
 import com.protomaps.basemap.feature.RegionInfos;
 import com.protomaps.basemap.names.NeNames;
 import com.protomaps.basemap.names.OsmNames;
@@ -23,6 +24,12 @@
 
 public class Places implements ForwardingProfile.FeatureProcessor, ForwardingProfile.FeaturePostProcessor {
 
+  private NaturalEarthDb naturalEarthDb;
+
+  public Places(NaturalEarthDb naturalEarthDb) {
+    this.naturalEarthDb = naturalEarthDb;
+  }
+
   @Override
   public String name() {
     return "places";

tiles/src/main/java/com/protomaps/basemap/layers/Pois.java

@@ -10,12 +10,19 @@
 import com.onthegomap.planetiler.reader.SourceFeature;
 import com.onthegomap.planetiler.util.ZoomFunction;
 import com.protomaps.basemap.feature.FeatureId;
+import com.protomaps.basemap.feature.QrankDb;
 import com.protomaps.basemap.names.OsmNames;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class Pois implements ForwardingProfile.FeatureProcessor, ForwardingProfile.FeaturePostProcessor {
 
+  private QrankDb qrankDb;
+
+  public Pois (QrankDb qrankDb) {
+    this.qrankDb = qrankDb;
+  }
+
   @Override
   public String name() {
     return "pois";
@@ -51,6 +58,16 @@ public void processFeature(SourceFeature sf, FeatureCollector features) {
       String kindDetail = "";
       Integer minZoom = 15;
 
+      String wikidata = sf.getString("wikidata");
+      if (wikidata != null) {
+        try {
+          long wikidataId = Long.parseLong(wikidata.substring(1));
+          qrankDb.get(wikidataId);
+        } catch (NumberFormatException e) {
+          // do nothing
+        }
+      }
+
       if (sf.hasTag("aeroway", "aerodrome")) {
         kind = sf.getString("aeroway");
         minZoom = 13;

tiles/src/test/java/com/protomaps/basemap/layers/LayerTest.java

@@ -18,7 +18,7 @@ abstract class LayerTest {
   final PlanetilerConfig params = PlanetilerConfig.defaults();
   final Stats stats = Stats.inMemory();
   final FeatureCollector.Factory featureCollectorFactory = new FeatureCollector.Factory(params, stats);
-  final Basemap profile = new Basemap(null, null);
+  final Basemap profile = new Basemap(null, null, null);
 
   static void assertFeatures(int zoom, List<Map<String, Object>> expected, Iterable<FeatureCollector.Feature> actual) {
     var expectedList = expected.stream().toList();