feat: Read country tags from an enriched PBF file

ors-api/src/test/files/borders/borders.csv

@@ -0,0 +1,5 @@
+iso;name_en;geom
+XC;Area 3 English;MULTIPOLYGON(((8.686269562020373 49.404812487369142,8.686034209438265 49.403748108474431,8.68550466612848 49.40093773101411,8.694024429601255 49.399995797403818,8.695024678075269 49.403617931054903,8.686269562020373 49.404812487369142)))
+XD;Area 4 English;MULTIPOLYGON(((8.686269562020376 49.404812487369142,8.687116831316011 49.407446534490731,8.696119067582131 49.406627238463642,8.695024678075269 49.403617931054896,8.686269562020376 49.404812487369142)))
+XA;Area 1 English;MULTIPOLYGON(((8.68603420943826 49.403748108474424,8.680209233030766 49.404582767258027,8.681739024814549 49.407913603676974,8.687116831316008 49.407446534490731,8.686269562020374 49.404812487369142,8.68603420943826 49.403748108474424)))
+XB;Area 2 English;MULTIPOLYGON(((8.680209233030766 49.404582767258027,8.679091308265694 49.401680543964602,8.685504666128484 49.400937731014132,8.686034209438256 49.403748108474439,8.680209233030766 49.404582767258027)))

ors-api/src/test/files/borders/ids.csv

@@ -1,5 +1,5 @@
 "country_id","name","name:en","iso_code_cca2","iso_code_cca3"
-"1","Area 1","Area 1 English","AT","AUT"
-"2","Area 2","Area 2 English","CH","CHE"
-"3","Area 3","Area 3 English","FR","FRA"
-"4","Area 4","Area 4 English","DE","DEU"
+"1","Area 1","Area 1 English","XA","XXA"
+"2","Area 2","Area 2 English","XB","XXB"
+"3","Area 3","Area 3 English","XC","XXC"
+"4","Area 4","Area 4 English","XD","XXD"

ors-api/src/test/java/org/heigit/ors/apitests/routing/ResultTest.java

@@ -1802,7 +1802,7 @@ void testBordersAvoid() {
                 .then()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(closeTo(1404, 1)))
+                .body("routes[0].summary.distance", is(closeTo(1156, 1)))
                 .statusCode(200);
 
         options = new JSONObject();
@@ -1846,7 +1846,7 @@ void testCountryExclusion() {
                 .then().log().ifValidationFails()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(closeTo(1156.6, 1)))
+                .body("routes[0].summary.distance", is(closeTo(1156, 1)))
                 .statusCode(200);
         options = new JSONObject();
         options.put("avoid_countries", constructFromPipedList("1|3"));
@@ -1862,11 +1862,11 @@ void testCountryExclusion() {
                 .then()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(closeTo(3172.4, 3)))
+                .body("routes[0].summary.distance", is(closeTo(3159, 3)))
                 .statusCode(200);
 
         // Test avoid_countries with ISO 3166-1 Alpha-2 parameters
-        options.put("avoid_countries", constructFromPipedList("AT|FR"));
+        options.put("avoid_countries", constructFromPipedList("XA|XC"));
         given()
                 .config(JSON_CONFIG_DOUBLE_NUMBERS)
                 .headers(CommonHeaders.jsonContent)
@@ -1877,11 +1877,11 @@ void testCountryExclusion() {
                 .then().log().ifValidationFails()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(closeTo(3172.4, 3)))
+                .body("routes[0].summary.distance", is(closeTo(3159, 3)))
                 .statusCode(200);
 
         // Test avoid_countries with ISO 3166-1 Alpha-3 parameters
-        options.put("avoid_countries", constructFromPipedList("AUT|FRA"));
+        options.put("avoid_countries", constructFromPipedList("XXA|XXC"));
         given()
                 .config(JSON_CONFIG_DOUBLE_NUMBERS)
                 .headers(CommonHeaders.jsonContent)
@@ -1892,7 +1892,7 @@ void testCountryExclusion() {
                 .then()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(closeTo(3172.4f, 3)))
+                .body("routes[0].summary.distance", is(closeTo(3159, 3)))
                 .statusCode(200);
 
     }
@@ -2018,7 +2018,7 @@ void testWheelchairWidthRestriction() {
                 .then()
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
-                .body("routes[0].summary.distance", is(158.7f))
+                .body("routes[0].summary.distance", is(158.8f))
                 .body("routes[0].summary.duration", is(114.3f))
                 .statusCode(200);
     }
@@ -2100,7 +2100,7 @@ void testWheelchairKerbRestriction() {
                 .assertThat()
                 .body("any { it.key == 'routes' }", is(true))
                 .body("routes[0].summary.distance", is(74.1f))
-                .body("routes[0].summary.duration", is(57.9f))
+                .body("routes[0].summary.duration", is(49.2f))
                 .statusCode(200);
 
         restrictions = new JSONObject();
@@ -3089,7 +3089,7 @@ void testCoutryTraversalCloseToBorder() {
         // Close to a border crossing
         given()
                 .headers(CommonHeaders.jsonContent)
-                .pathParam("profile", getParameter("carProfile"))
+                .pathParam("profile", "driving-hgv")
                 .body(body.toString())
                 .when()
                 .post(getEndPointPath() + "/{profile}/json")

ors-api/src/test/resources/application-test.yml

@@ -12,7 +12,7 @@ ors:
       maximum_intervals: 10
 
   engine:
-    source_file: ./src/test/files/heidelberg.osm.gz
+    source_file: ./src/test/files/heidelberg.ors.pbf
     graphs_root_path: graphs-apitests
     elevation:
       cache_path: ./src/test/files/elevation
@@ -68,7 +68,7 @@ ors:
           WaySurfaceType:
           Tollways:
           Borders:
-            boundaries: ./src/test/files/borders/borders.geojson
+            preprocessed: true
             ids: ./src/test/files/borders/ids.csv
             openborders: ./src/test/files/borders/openborders.csv
           RoadAccessRestrictions:

ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/ORSOSMReader.java

@@ -14,6 +14,7 @@
 package org.heigit.ors.routing.graphhopper.extensions;
 
 import com.carrotsearch.hppc.LongArrayList;
+import com.graphhopper.coll.GHLongObjectHashMap;
 import com.graphhopper.reader.ReaderNode;
 import com.graphhopper.reader.ReaderWay;
 import com.graphhopper.reader.osm.OSMReader;
@@ -38,9 +39,9 @@ public class ORSOSMReader extends OSMReader {
     private final GraphProcessContext procCntx;
     private boolean processNodeTags;
     private final OSMDataReaderContext readerCntx;
-
-    private final HashMap<Long, HashMap<String, String>> nodeTags = new HashMap<>();
-
+    private static final String KEY_COUNTRY = "country";
+    private Map<Long, String> countries;
+    private final GHLongObjectHashMap<Map<String, String>> nodeTags = new GHLongObjectHashMap<>(200, 0.5);
     private boolean processGeom = false;
     private boolean processSimpleGeom = false;
     private boolean processWholeGeom = false;
@@ -67,6 +68,7 @@ public ORSOSMReader(GraphHopperStorage storage, GraphProcessContext procCntx) {
         // Look if we should do border processing - if so then we have to process the geometry
         for (GraphStorageBuilder b : this.procCntx.getStorageBuilders()) {
             if (b instanceof BordersGraphStorageBuilder) {
+                this.countries = new HashMap<>();
                 this.processGeom = true;
             }
 
@@ -123,6 +125,11 @@ public ReaderNode onProcessNode(ReaderNode node) {
                 nodeTags.put(node.getId(), tagValues);
             }
         }
+
+        if (countries != null  && node.hasTag(KEY_COUNTRY)) {
+            countries.put(node.getId(), node.getTag(KEY_COUNTRY));
+        }
+
         return node;
     }
 
@@ -169,7 +176,6 @@ protected void processWay(ReaderWay way) {
      */
     @Override
     public void onProcessWay(ReaderWay way) {
-
         Map<Integer, Map<String, String>> tags = new HashMap<>();
         ArrayList<Coordinate> coords = new ArrayList<>();
         ArrayList<Coordinate> allCoordinates = new ArrayList<>();
@@ -187,10 +193,16 @@ public void onProcessWay(ReaderWay way) {
                 long id = osmNodeIds.get(i);
                 // replace the osm id with the internal id
                 int internalId = getNodeMap().get(id);
-                HashMap<String, String> tagsForNode = nodeTags.get(id);
+                Map<String, String> tagsForNode = nodeTags.get(id);
+
+                if (countries != null && countries.containsKey(id)) {
+                    if (tagsForNode == null)
+                        tagsForNode = new HashMap<>();
+                    tagsForNode.put(KEY_COUNTRY, countries.get(id));
+                }
 
                 if (tagsForNode != null) {
-                    tags.put(internalId, nodeTags.get(id));
+                    tags.put(internalId, tagsForNode);
                 }
             }
         }

ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/reader/borders/CountryBordersReader.java

@@ -33,18 +33,17 @@ public class CountryBordersReader {
     public static final String INTERNATIONAL_NAME = "INTERNATIONAL";
     public static final String INTERNATIONAL_ID = "-1";
     public static final String KEY_PROPERTIES = "properties";
+    private static final String NAME_FIELD = "name";
+    private static final String HIERARCHY_ID_FIELD = "hierarchy";
 
     private final String borderFile;
-    private final String nameField;
-    private final String hierarchyIdField;
-
     private final String idsPath;
     private final String openPath;
 
     private final HashMap<String, CountryInfo> ids = new HashMap<>();
     private final HashMap<String, ArrayList<String>> openBorders = new HashMap<>();
-    private final HashMap<String, Integer> isoCodes = new HashMap<>();
-
+    private final Map<String, Short> isoCodes = new HashMap<>();
+    private Map<Short, String> names = new HashMap<>();
     private final HashMap<Long, CountryBordersHierarchy> hierarchies = new HashMap<>();
 
     // Package scoped for testing purposes
@@ -55,8 +54,6 @@ public class CountryBordersReader {
      */
     public CountryBordersReader() {
         borderFile = "";
-        nameField = "name";
-        hierarchyIdField = "hierarchy";
         idsPath = "";
         openPath = "";
 
@@ -72,17 +69,16 @@ public CountryBordersReader() {
      */
     public CountryBordersReader(String filepath, String idsPath, String openPath) throws IOException {
         borderFile = filepath;
-        nameField = "name";
-        hierarchyIdField = "hierarchy";
-
         this.idsPath = idsPath;
         this.openPath = openPath;
 
         try {
-            JSONObject data = readBordersData();
-            LOGGER.info("Border geometries read");
+            if (!"".equals(borderFile)) {
+                JSONObject data = readBordersData();
+                LOGGER.info("Border geometries read");
 
-            createGeometries(data);
+                createGeometries(data);
+            }
 
             readIds();
             LOGGER.info("Border ids data read");
@@ -108,8 +104,8 @@ public void addHierarchy(Long id, CountryBordersHierarchy hierarchy) {
     public void addId(String id, String localName, String englishName, String cca2, String cca3) {
         if (!ids.containsKey(localName)) {
             ids.put(localName, new CountryInfo(id, localName, englishName));
-            isoCodes.put(cca2.trim().toUpperCase(), Integer.parseInt(id));
-            isoCodes.put(cca3.trim().toUpperCase(), Integer.parseInt(id));
+            isoCodes.put(cca2.trim().toUpperCase(), Short.parseShort(id));
+            isoCodes.put(cca3.trim().toUpperCase(), Short.parseShort(id));
         }
     }
 
@@ -228,13 +224,13 @@ private void createGeometries(JSONObject json) {
                 Geometry geom = GeometryJSON.parse(obj.getJSONObject("geometry"));
 
                 // Also need the id of the country and its hierarchy id
-                String id = obj.getJSONObject(KEY_PROPERTIES).getString(nameField);
+                String id = obj.getJSONObject(KEY_PROPERTIES).getString(NAME_FIELD);
 
                 Long hId = -1L;
 
                 // If there is no hierarchy info, then we set the id of the hierarchy to be a default of 1
-                if (obj.getJSONObject(KEY_PROPERTIES).has(hierarchyIdField))
-                    hId = obj.getJSONObject(KEY_PROPERTIES).getLong(hierarchyIdField);
+                if (obj.getJSONObject(KEY_PROPERTIES).has(HIERARCHY_ID_FIELD))
+                    hId = obj.getJSONObject(KEY_PROPERTIES).getLong(HIERARCHY_ID_FIELD);
 
                 // Create the borders object
                 CountryBordersPolygon c = new CountryBordersPolygon(id, geom);
@@ -346,6 +342,10 @@ public String getEngName(String name) {
             return "";
     }
 
+    public String getName(short id) {
+        return names.get(id);
+    }
+
     /**
      * Get whether a border between two specified countries is open or closed
      *
@@ -370,8 +370,8 @@ public boolean isOpen(String c1, String c2) {
      * @param code The code to look up
      * @return The ID of the country or 0 if not found
      */
-    public static int getCountryIdByISOCode(String code) {
-        return currentInstance != null ? currentInstance.isoCodes.getOrDefault(code.toUpperCase(), 0) : 0;
+    public static short getCountryIdByISOCode(String code) {
+        return currentInstance != null ? currentInstance.isoCodes.getOrDefault(code.toUpperCase(), (short) 0) : 0;
     }
 
     /**
@@ -388,23 +388,24 @@ private void readIds() {
         int isoCCA2 = 0;
         int isoCCA3 = 0;
         for (List<String> col : data) {
-            if (col.size() >= 3) {
-                ids.put(col.get(1), new CountryInfo(col.get(0), col.get(1), col.get(2)));
-                countries++;
-            }
-            int intID = 0;
+            short shortID = 0;
             try {
-                intID = Integer.parseInt(col.get(0));
+                shortID = Short.parseShort(col.get(0));
             } catch (NumberFormatException e) {
                 LOGGER.error("Invalid country ID " + col.get(0));
                 continue;
             }
+            if (col.size() >= 3) {
+                ids.put(col.get(1), new CountryInfo(col.get(0), col.get(1), col.get(2)));
+                names.put(shortID, col.get(2));
+                countries++;
+            }
             if (col.size() >= 4 && !col.get(3).trim().isEmpty()) {
-                isoCodes.put(col.get(3).trim().toUpperCase(), intID);
+                isoCodes.put(col.get(3).trim().toUpperCase(), shortID);
                 isoCCA2++;
             }
             if (col.size() == 5 && !col.get(4).trim().isEmpty()) {
-                isoCodes.put(col.get(4).trim().toUpperCase(), intID);
+                isoCodes.put(col.get(4).trim().toUpperCase(), shortID);
                 isoCCA3++;
             }
         }