diff --git a/include/coordinates.h b/include/coordinates.h index 3ca3b8cf..6d34ca61 100644 --- a/include/coordinates.h +++ b/include/coordinates.h @@ -19,6 +19,7 @@ typedef uint32_t ShardedNodeID; typedef uint64_t NodeID; typedef uint64_t WayID; +typedef uint64_t RelationID; typedef std::vector WayVec; diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index 52ab2e55..d21b83b0 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -81,6 +81,7 @@ class OsmLuaProcessing { // Do we have Lua routines for non-MP relations? bool canReadRelations(); + bool canPostScanRelations(); bool canWriteRelations(); // Shapefile tag remapping @@ -95,6 +96,9 @@ class OsmLuaProcessing { // Scan non-MP relation bool scanRelation(WayID id, const TagMap& tags); + // Post-scan non-MP relations + void postScanRelations(); + /// \brief We are now processing a significant node bool setNode(NodeID id, LatpLon node, const TagMap& tags); @@ -120,6 +124,15 @@ class OsmLuaProcessing { // Get the ID of the current object std::string Id() const; + // Check if there's a value for a given key + bool Holds(const std::string& key) const; + + // Get an OSM tag for a given key (or return empty string if none) + const std::string Find(const std::string& key) const; + + // Check if an object has any tags + bool HasTags() const; + // ---- Spatial queries called from Lua // Find intersecting shapefile layer @@ -203,6 +216,7 @@ class OsmLuaProcessing { void RestartRelations(); std::string FindInRelation(const std::string &key); void Accept(); + void SetTag(const std::string &key, const std::string &value); // Write error if in verbose mode void ProcessingError(const std::string &errStr) { @@ -248,6 +262,9 @@ class OsmLuaProcessing { relationList.clear(); relationSubscript = -1; lastStoredGeometryId = 0; + isWay = false; + isRelation = false; + isPostScanRelation = false; } void removeAttributeIfNeeded(const std::string& key); @@ -261,6 +278,7 @@ class OsmLuaProcessing { kaguya::State luaState; bool supportsRemappingShapefiles; bool supportsReadingRelations; + bool supportsPostScanRelations; bool supportsWritingRelations; const class ShpMemTiles &shpMemTiles; class OsmMemTiles &osmMemTiles; @@ -272,6 +290,7 @@ class OsmLuaProcessing { bool relationAccepted; // in scanRelation, whether we're using a non-MP relation std::vector> relationList; // in processNode/processWay, list of relations this entity is in, and its role int relationSubscript = -1; // in processWay, position in the relation list + bool isPostScanRelation; // processing a relation in postScanRelation int32_t lon,latp; ///< Node coordinates LatpLonVec const *llVecPtr; @@ -296,6 +315,7 @@ class OsmLuaProcessing { std::vector> outputs; // All output objects that have been created std::vector outputKeys; const PbfReader::Relation* currentRelation; + const boost::container::flat_map* currentPostScanTags; // for postScan only const std::vector* stringTable; std::vector finalizeOutputs(); diff --git a/include/osm_store.h b/include/osm_store.h index 0c204637..de86e367 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -97,31 +97,44 @@ class RelationScanStore { private: using tag_map_t = boost::container::flat_map; - std::vector>>> relationsForWays; - std::vector>>> relationsForNodes; - std::vector> relationTags; + std::vector>>> relationsForWays; + std::vector>>> relationsForNodes; + std::vector> relationTags; mutable std::vector mutex; RelationRoles relationRoles; public: + std::map>> relationsForRelations; + RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} - void relation_contains_way(WayID relid, WayID wayid, std::string role) { + + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = wayid % mutex.size(); std::lock_guard lock(mutex[shard]); relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); } - void relation_contains_node(WayID relid, NodeID nodeId, std::string role) { + void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); const size_t shard = nodeId % mutex.size(); std::lock_guard lock(mutex[shard]); relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); } - void store_relation_tags(WayID relid, const tag_map_t &tags) { + void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { + uint16_t roleId = relationRoles.getOrAddRole(role); + std::lock_guard lock(mutex[0]); + relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + } + void store_relation_tags(RelationID relid, const tag_map_t &tags) { const size_t shard = relid % mutex.size(); std::lock_guard lock(mutex[shard]); relationTags[shard][relid] = tags; } + void set_relation_tag(RelationID relid, const std::string &key, const std::string &value) { + const size_t shard = relid % mutex.size(); + std::lock_guard lock(mutex[shard]); + relationTags[shard][relid][key] = value; + } bool way_in_any_relations(WayID wayid) { const size_t shard = wayid % mutex.size(); return relationsForWays[shard].find(wayid) != relationsForWays[shard].end(); @@ -130,16 +143,47 @@ class RelationScanStore { const size_t shard = nodeId % mutex.size(); return relationsForNodes[shard].find(nodeId) != relationsForNodes[shard].end(); } + bool relation_in_any_relations(RelationID relId) { + return relationsForRelations.find(relId) != relationsForRelations.end(); + } std::string getRole(uint16_t roleId) const { return relationRoles.getRole(roleId); } const std::vector>& relations_for_way(WayID wayid) { const size_t shard = wayid % mutex.size(); return relationsForWays[shard][wayid]; } - const std::vector>& relations_for_node(NodeID nodeId) { + const std::vector>& relations_for_node(NodeID nodeId) { const size_t shard = nodeId % mutex.size(); return relationsForNodes[shard][nodeId]; } - std::string get_relation_tag(WayID relid, const std::string &key) { + const std::vector>& relations_for_relation(RelationID relId) { + return relationsForRelations[relId]; + } + const tag_map_t& relation_tags(RelationID relId) { + const size_t shard = relId % mutex.size(); + return relationTags[shard][relId]; + } + // return all the parent relations (and their parents &c.) for a given relation + std::vector> relations_for_relation_with_parents(RelationID relId) { + std::vector relationsToDo; + std::set relationsDone; + std::vector> out; + relationsToDo.emplace_back(relId); + // check parents in turn, pushing onto the stack if necessary + while (!relationsToDo.empty()) { + RelationID rel = relationsToDo.back(); + relationsToDo.pop_back(); + // check it's not already been added + if (relationsDone.find(rel) != relationsDone.end()) continue; + relationsDone.insert(rel); + // add all its parents + for (auto rp : relationsForRelations[rel]) { + out.emplace_back(rp); + relationsToDo.emplace_back(rp.first); + } + } + return out; + } + std::string get_relation_tag(RelationID relid, const std::string &key) { const size_t shard = relid % mutex.size(); auto it = relationTags[shard].find(relid); if (it==relationTags[shard].end()) return ""; diff --git a/include/tag_map.h b/include/tag_map.h index 1d29ef26..be5fa97c 100644 --- a/include/tag_map.h +++ b/include/tag_map.h @@ -33,7 +33,7 @@ class TagMap { TagMap(); void reset(); - bool empty(); + bool empty() const; void addTag(const protozero::data_view& key, const protozero::data_view& value); // Return -1 if key not found, else return its keyLoc. diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index 4e0c5478..cf95bc79 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -16,6 +16,7 @@ using namespace std; const std::string EMPTY_STRING = ""; thread_local kaguya::State *g_luaState = nullptr; thread_local OsmLuaProcessing* osmLuaProcessing = nullptr; +thread_local bool inPostScanRelations = false; void handleOsmLuaProcessingUserSignal(int signum) { osmLuaProcessing->handleUserSignal(signum); @@ -42,6 +43,11 @@ thread_local Sigusr1Handler sigusr1Handler; struct KnownTagKey { bool found; uint32_t index; + + // stringValue is populated only in PostScanRelations phase; we could consider + // having osm_store's relationTags use TagMap, in which case we'd be able to + // use the found/index fields + std::string stringValue; }; template<> struct kaguya::lua_type_traits { @@ -62,6 +68,12 @@ template<> struct kaguya::lua_type_traits { size_t size = 0; const char* buffer = lua_tolstring(l, index, &size); + if (inPostScanRelations) { + rv.stringValue = std::string(buffer, size); + return rv; + } + + int64_t tagLoc = osmLuaProcessing->currentTags->getKey(buffer, size); if (tagLoc >= 0) { @@ -123,6 +135,9 @@ template<> struct kaguya::lua_type_traits { std::string rawId() { return osmLuaProcessing->Id(); } bool rawHolds(const KnownTagKey& key) { return key.found; } +bool rawHoldsPostScanRelations(const KnownTagKey& key) { return key.found; } +bool rawHasTags() { return osmLuaProcessing->HasTags(); } +void rawSetTag(const std::string &key, const std::string &value) { return osmLuaProcessing->SetTag(key, value); } const std::string rawFind(const KnownTagKey& key) { if (key.found) { auto value = *(osmLuaProcessing->currentTags->getValueFromKey(key.index)); @@ -131,6 +146,9 @@ const std::string rawFind(const KnownTagKey& key) { return EMPTY_STRING; } +const std::string rawFindPostScanRelations(const KnownTagKey& key) { + return osmLuaProcessing->Find(key.stringValue); +} std::vector rawFindIntersecting(const std::string &layerName) { return osmLuaProcessing->FindIntersecting(layerName); } bool rawIntersects(const std::string& layerName) { return osmLuaProcessing->Intersects(layerName); } std::vector rawFindCovering(const std::string& layerName) { return osmLuaProcessing->FindCovering(layerName); } @@ -191,6 +209,8 @@ OsmLuaProcessing::OsmLuaProcessing( luaState["Id"] = &rawId; luaState["Holds"] = &rawHolds; luaState["Find"] = &rawFind; + luaState["HasTags"] = &rawHasTags; + luaState["SetTag"] = &rawSetTag; luaState["FindIntersecting"] = &rawFindIntersecting; luaState["Intersects"] = &rawIntersects; luaState["FindCovering"] = &rawFindCovering; @@ -223,6 +243,7 @@ OsmLuaProcessing::OsmLuaProcessing( luaState["FindInRelation"] = &rawFindInRelation; supportsRemappingShapefiles = !!luaState["attribute_function"]; supportsReadingRelations = !!luaState["relation_scan_function"]; + supportsPostScanRelations = !!luaState["relation_postscan_function"]; supportsWritingRelations = !!luaState["relation_function"]; // ---- Call init_function of Lua logic @@ -256,6 +277,10 @@ bool OsmLuaProcessing::canReadRelations() { return supportsReadingRelations; } +bool OsmLuaProcessing::canPostScanRelations() { + return supportsPostScanRelations; +} + bool OsmLuaProcessing::canWriteRelations() { return supportsWritingRelations; } @@ -276,6 +301,25 @@ string OsmLuaProcessing::Id() const { return to_string(originalOsmID); } +// Check if there's a value for a given key +bool OsmLuaProcessing::Holds(const string& key) const { + // NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawHolds + return currentPostScanTags->find(key)!=currentPostScanTags->end(); +} + +// Get an OSM tag for a given key (or return empty string if none) +const string OsmLuaProcessing::Find(const string& key) const { + // NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawFind + auto it = currentPostScanTags->find(key); + if (it == currentPostScanTags->end()) return EMPTY_STRING; + return it->second; +} + +// Check if an object has any tags +bool OsmLuaProcessing::HasTags() const { + return isPostScanRelation ? !currentPostScanTags->empty() : !currentTags->empty(); +} + // ---- Spatial queries called from Lua vector OsmLuaProcessing::FindIntersecting(const string &layerName) { @@ -771,6 +815,11 @@ std::vector OsmLuaProcessing::Centroid(kaguya::VariadicArgType algorithm void OsmLuaProcessing::Accept() { relationAccepted = true; } +// Set a tag in post-scan phase +void OsmLuaProcessing::SetTag(const std::string &key, const std::string &value) { + if (!isPostScanRelation) throw std::runtime_error("SetTag can only be used in relation_postscan_function"); + osmStore.scannedRelations.set_relation_tag(originalOsmID, key, value); +} void OsmLuaProcessing::removeAttributeIfNeeded(const string& key) { // Does it exist? @@ -882,7 +931,6 @@ void OsmLuaProcessing::setVectorLayerMetadata(const uint_least8_t layer, const s bool OsmLuaProcessing::scanRelation(WayID id, const TagMap& tags) { reset(); originalOsmID = id; - isWay = false; isRelation = true; currentTags = &tags; try { @@ -899,11 +947,29 @@ bool OsmLuaProcessing::scanRelation(WayID id, const TagMap& tags) { return true; } +// Post-scan relations - typically used for bouncing down values from nested relations +void OsmLuaProcessing::postScanRelations() { + if (!supportsPostScanRelations) return; + + // Adjust the function pointers for tag-related functions + inPostScanRelations = true; + luaState["Holds"] = &rawHoldsPostScanRelations; + luaState["Find"] = &rawFindPostScanRelations; + + for (const auto &relp : osmStore.scannedRelations.relationsForRelations) { + reset(); + isPostScanRelation = true; + RelationID id = relp.first; + originalOsmID = id; + currentPostScanTags = &(osmStore.scannedRelations.relation_tags(id)); + relationList = osmStore.scannedRelations.relations_for_relation_with_parents(id); + luaState["relation_postscan_function"](this); + } +} + bool OsmLuaProcessing::setNode(NodeID id, LatpLon node, const TagMap& tags) { reset(); originalOsmID = id; - isWay = false; - isRelation = false; lon = node.lon; latp= node.latp; currentTags = &tags; @@ -939,7 +1005,6 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap wayEmitted = false; originalOsmID = wayId; isWay = true; - isRelation = false; llVecPtr = &llVec; outerWayVecPtr = nullptr; innerWayVecPtr = nullptr; @@ -1004,6 +1069,10 @@ void OsmLuaProcessing::setRelation( innerWayVecPtr = &innerWayVec; currentTags = &tags; + if (supportsReadingRelations && osmStore.scannedRelations.relation_in_any_relations(originalOsmID)) { + relationList = osmStore.scannedRelations.relations_for_relation(originalOsmID); + } + // Start Lua processing for relation if (!isNativeMP && !supportsWritingRelations) return; try { diff --git a/src/pbf_processor.cpp b/src/pbf_processor.cpp index 0e2a1fd7..4f6ac2c5 100644 --- a/src/pbf_processor.cpp +++ b/src/pbf_processor.cpp @@ -221,6 +221,12 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG std::string role(roleView.data(), roleView.size()); osmStore.scannedRelations.relation_contains_node(relid, lastID, role); } + } else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::RELATION) { + if (isAccepted) { + const auto& roleView = pb.stringTable[pbfRelation.roles_sid[n]]; + std::string role(roleView.data(), roleView.size()); + osmStore.scannedRelations.relation_contains_relation(relid, lastID, role); + } } else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::WAY) { if (lastID >= pow(2,42)) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID)); osmStore.mark_way_used(static_cast(lastID)); @@ -674,6 +680,10 @@ int PbfProcessor::ReadPbfFile( #endif } + if(phase == ReadPhase::RelationScan) { + auto output = generate_output(); + output->postScanRelations(); + } if(phase == ReadPhase::Nodes) { osmStore.nodes.finalize(threadNum); osmStore.usedNodes.clear(); diff --git a/src/tag_map.cpp b/src/tag_map.cpp index b93b502f..c3424fb7 100644 --- a/src/tag_map.cpp +++ b/src/tag_map.cpp @@ -16,7 +16,7 @@ void TagMap::reset() { } } -bool TagMap::empty() { +bool TagMap::empty() const { for (int i = 0; i < keys.size(); i++) if (keys[i].size() > 0) return false;