Merge remote-tracking branch 'origin/nested_relations' into v3-post-r…

…elation-scan

I don't think this is ready to merge yet; I might be misunderstanding
something.

include/coordinates.h

@@ -19,6 +19,7 @@
 typedef uint32_t ShardedNodeID;
 typedef uint64_t NodeID;
 typedef uint64_t WayID;
+typedef uint64_t RelationID;
 
 typedef std::vector<WayID> WayVec;
 

include/osm_lua_processing.h

@@ -81,6 +81,7 @@ class OsmLuaProcessing {
 
 	// Do we have Lua routines for non-MP relations?
 	bool canReadRelations();
+	bool canPostScanRelations();
 	bool canWriteRelations();
 
 	// Shapefile tag remapping
@@ -95,6 +96,9 @@ class OsmLuaProcessing {
 	// Scan non-MP relation
 	bool scanRelation(WayID id, const TagMap& tags);
 
+	// Post-scan non-MP relations
+	void postScanRelations();
+
 	/// \brief We are now processing a significant node
 	bool setNode(NodeID id, LatpLon node, const TagMap& tags);
 
@@ -120,6 +124,15 @@ class OsmLuaProcessing {
 	// Get the ID of the current object
 	std::string Id() const;
 
+	// Check if there's a value for a given key
+	bool Holds(const std::string& key) const;
+
+	// Get an OSM tag for a given key (or return empty string if none)
+	const std::string Find(const std::string& key) const;
+
+	// Check if an object has any tags
+	bool HasTags() const;
+
 	// ----	Spatial queries called from Lua
 
 	// Find intersecting shapefile layer
@@ -203,6 +216,7 @@ class OsmLuaProcessing {
 	void RestartRelations();
 	std::string FindInRelation(const std::string &key);
 	void Accept();
+	void SetTag(const std::string &key, const std::string &value);
 
 	// Write error if in verbose mode
 	void ProcessingError(const std::string &errStr) {
@@ -248,6 +262,9 @@ class OsmLuaProcessing {
 		relationList.clear();
 		relationSubscript = -1;
 		lastStoredGeometryId = 0;
+		isWay = false;
+		isRelation = false;
+		isPostScanRelation = false;
 	}
 
 	void removeAttributeIfNeeded(const std::string& key);
@@ -261,6 +278,7 @@ class OsmLuaProcessing {
 	kaguya::State luaState;
 	bool supportsRemappingShapefiles;
 	bool supportsReadingRelations;
+	bool supportsPostScanRelations;
 	bool supportsWritingRelations;
 	const class ShpMemTiles &shpMemTiles;
 	class OsmMemTiles &osmMemTiles;
@@ -272,6 +290,7 @@ class OsmLuaProcessing {
 	bool relationAccepted;					// in scanRelation, whether we're using a non-MP relation
 	std::vector<std::pair<WayID, uint16_t>> relationList;		// in processNode/processWay, list of relations this entity is in, and its role
 	int relationSubscript = -1;				// in processWay, position in the relation list
+	bool isPostScanRelation;				// processing a relation in postScanRelation
 
 	int32_t lon,latp;						///< Node coordinates
 	LatpLonVec const *llVecPtr;
@@ -296,6 +315,7 @@ class OsmLuaProcessing {
 	std::vector<std::pair<OutputObject, AttributeSet>> outputs;		// All output objects that have been created
 	std::vector<std::string> outputKeys;
 	const PbfReader::Relation* currentRelation;
+	const boost::container::flat_map<std::string, std::string>* currentPostScanTags; // for postScan only
 	const std::vector<protozero::data_view>* stringTable;
 
 	std::vector<OutputObject> finalizeOutputs();

include/osm_store.h

@@ -97,31 +97,44 @@ class RelationScanStore {
 
 private:
 	using tag_map_t = boost::container::flat_map<std::string, std::string>;
-	std::vector<std::map<WayID, std::vector<std::pair<WayID, uint16_t>>>> relationsForWays;
-	std::vector<std::map<NodeID, std::vector<std::pair<WayID, uint16_t>>>> relationsForNodes;
-	std::vector<std::map<WayID, tag_map_t>> relationTags;
+	std::vector<std::map<WayID, std::vector<std::pair<RelationID, uint16_t>>>> relationsForWays;
+	std::vector<std::map<NodeID, std::vector<std::pair<RelationID, uint16_t>>>> relationsForNodes;
+	std::vector<std::map<RelationID, tag_map_t>> relationTags;
 	mutable std::vector<std::mutex> mutex;
 	RelationRoles relationRoles;
 
 public:
+	std::map<RelationID, std::vector<std::pair<RelationID, uint16_t>>> relationsForRelations;
+
 	RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {}
-	void relation_contains_way(WayID relid, WayID wayid, std::string role) {
+
+	void relation_contains_way(RelationID relid, WayID wayid, std::string role) {
 		uint16_t roleId = relationRoles.getOrAddRole(role);
 		const size_t shard = wayid % mutex.size();
 		std::lock_guard<std::mutex> lock(mutex[shard]);
 		relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId));
 	}
-	void relation_contains_node(WayID relid, NodeID nodeId, std::string role) {
+	void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) {
 		uint16_t roleId = relationRoles.getOrAddRole(role);
 		const size_t shard = nodeId % mutex.size();
 		std::lock_guard<std::mutex> lock(mutex[shard]);
 		relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId));
 	}
-	void store_relation_tags(WayID relid, const tag_map_t &tags) {
+	void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) {
+		uint16_t roleId = relationRoles.getOrAddRole(role);
+		std::lock_guard<std::mutex> lock(mutex[0]);
+		relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId));
+	}
+	void store_relation_tags(RelationID relid, const tag_map_t &tags) {
 		const size_t shard = relid % mutex.size();
 		std::lock_guard<std::mutex> lock(mutex[shard]);
 		relationTags[shard][relid] = tags;
 	}
+	void set_relation_tag(RelationID relid, const std::string &key, const std::string &value) {
+		const size_t shard = relid % mutex.size();
+		std::lock_guard<std::mutex> lock(mutex[shard]);
+		relationTags[shard][relid][key] = value;
+	}
 	bool way_in_any_relations(WayID wayid) {
 		const size_t shard = wayid % mutex.size();
 		return relationsForWays[shard].find(wayid) != relationsForWays[shard].end();
@@ -130,16 +143,47 @@ class RelationScanStore {
 		const size_t shard = nodeId % mutex.size();
 		return relationsForNodes[shard].find(nodeId) != relationsForNodes[shard].end();
 	}
+	bool relation_in_any_relations(RelationID relId) {
+		return relationsForRelations.find(relId) != relationsForRelations.end();
+	}
 	std::string getRole(uint16_t roleId) const { return relationRoles.getRole(roleId); }
 	const std::vector<std::pair<WayID, uint16_t>>& relations_for_way(WayID wayid) {
 		const size_t shard = wayid % mutex.size();
 		return relationsForWays[shard][wayid];
 	}
-	const std::vector<std::pair<WayID, uint16_t>>& relations_for_node(NodeID nodeId) {
+	const std::vector<std::pair<RelationID, uint16_t>>& relations_for_node(NodeID nodeId) {
 		const size_t shard = nodeId % mutex.size();
 		return relationsForNodes[shard][nodeId];
 	}
-	std::string get_relation_tag(WayID relid, const std::string &key) {
+	const std::vector<std::pair<RelationID, uint16_t>>& relations_for_relation(RelationID relId) {
+		return relationsForRelations[relId];
+	}
+	const tag_map_t& relation_tags(RelationID relId) {
+		const size_t shard = relId % mutex.size();
+		return relationTags[shard][relId];
+	}
+	// return all the parent relations (and their parents &c.) for a given relation
+	std::vector<std::pair<RelationID, uint16_t>> relations_for_relation_with_parents(RelationID relId) {
+		std::vector<RelationID> relationsToDo;
+		std::set<RelationID> relationsDone;
+		std::vector<std::pair<WayID, uint16_t>> out;
+		relationsToDo.emplace_back(relId);
+		// check parents in turn, pushing onto the stack if necessary
+		while (!relationsToDo.empty()) {
+			RelationID rel = relationsToDo.back();
+			relationsToDo.pop_back();
+			// check it's not already been added
+			if (relationsDone.find(rel) != relationsDone.end()) continue;
+			relationsDone.insert(rel);
+			// add all its parents
+			for (auto rp : relationsForRelations[rel]) {
+				out.emplace_back(rp);
+				relationsToDo.emplace_back(rp.first);
+			}
+		}
+		return out;
+	}
+	std::string get_relation_tag(RelationID relid, const std::string &key) {
 		const size_t shard = relid % mutex.size();
 		auto it = relationTags[shard].find(relid);
 		if (it==relationTags[shard].end()) return "";

include/tag_map.h

@@ -33,7 +33,7 @@ class TagMap {
 	TagMap();
 	void reset();
 
-	bool empty();
+	bool empty() const;
 	void addTag(const protozero::data_view& key, const protozero::data_view& value);
 
 	// Return -1 if key not found, else return its keyLoc.

src/osm_lua_processing.cpp

@@ -16,6 +16,7 @@ using namespace std;
 const std::string EMPTY_STRING = "";
 thread_local kaguya::State *g_luaState = nullptr;
 thread_local OsmLuaProcessing* osmLuaProcessing = nullptr;
+thread_local bool inPostScanRelations = false;
 
 void handleOsmLuaProcessingUserSignal(int signum) {
 	osmLuaProcessing->handleUserSignal(signum);
@@ -42,6 +43,11 @@ thread_local Sigusr1Handler sigusr1Handler;
 struct KnownTagKey {
 	bool found;
 	uint32_t index;
+
+	// stringValue is populated only in PostScanRelations phase; we could consider
+	// having osm_store's relationTags use TagMap, in which case we'd be able to
+	// use the found/index fields
+	std::string stringValue;
 };
 
 template<>  struct kaguya::lua_type_traits<KnownTagKey> {
@@ -62,6 +68,12 @@ template<>  struct kaguya::lua_type_traits<KnownTagKey> {
 		size_t size = 0;
 		const char* buffer = lua_tolstring(l, index, &size);
 
+		if (inPostScanRelations) {
+			rv.stringValue = std::string(buffer, size);
+			return rv;
+		}
+
+
 		int64_t tagLoc = osmLuaProcessing->currentTags->getKey(buffer, size);
 
 		if (tagLoc >= 0) {
@@ -123,6 +135,9 @@ template<>  struct kaguya::lua_type_traits<PossiblyKnownTagValue> {
 
 std::string rawId() { return osmLuaProcessing->Id(); }
 bool rawHolds(const KnownTagKey& key) { return key.found; }
+bool rawHoldsPostScanRelations(const KnownTagKey& key) { return key.found; }
+bool rawHasTags() { return osmLuaProcessing->HasTags(); }
+void rawSetTag(const std::string &key, const std::string &value) { return osmLuaProcessing->SetTag(key, value); }
 const std::string rawFind(const KnownTagKey& key) {
 	if (key.found) {
 		auto value = *(osmLuaProcessing->currentTags->getValueFromKey(key.index));
@@ -131,6 +146,9 @@ const std::string rawFind(const KnownTagKey& key) {
 
 	return EMPTY_STRING;
 }
+const std::string rawFindPostScanRelations(const KnownTagKey& key) {
+	return osmLuaProcessing->Find(key.stringValue);
+}
 std::vector<std::string> rawFindIntersecting(const std::string &layerName) { return osmLuaProcessing->FindIntersecting(layerName); }
 bool rawIntersects(const std::string& layerName) { return osmLuaProcessing->Intersects(layerName); }
 std::vector<std::string> rawFindCovering(const std::string& layerName) { return osmLuaProcessing->FindCovering(layerName); }
@@ -191,6 +209,8 @@ OsmLuaProcessing::OsmLuaProcessing(
 	luaState["Id"] = &rawId;
 	luaState["Holds"] = &rawHolds;
 	luaState["Find"] = &rawFind;
+	luaState["HasTags"] = &rawHasTags;
+	luaState["SetTag"] = &rawSetTag;
 	luaState["FindIntersecting"] = &rawFindIntersecting;
 	luaState["Intersects"] = &rawIntersects;
 	luaState["FindCovering"] = &rawFindCovering;
@@ -223,6 +243,7 @@ OsmLuaProcessing::OsmLuaProcessing(
 	luaState["FindInRelation"] = &rawFindInRelation;
 	supportsRemappingShapefiles = !!luaState["attribute_function"];
 	supportsReadingRelations    = !!luaState["relation_scan_function"];
+	supportsPostScanRelations   = !!luaState["relation_postscan_function"];
 	supportsWritingRelations    = !!luaState["relation_function"];
 
 	// ---- Call init_function of Lua logic
@@ -256,6 +277,10 @@ bool OsmLuaProcessing::canReadRelations() {
 	return supportsReadingRelations;
 }
 
+bool OsmLuaProcessing::canPostScanRelations() {
+	return supportsPostScanRelations;
+}
+
 bool OsmLuaProcessing::canWriteRelations() {
 	return supportsWritingRelations;
 }
@@ -276,6 +301,25 @@ string OsmLuaProcessing::Id() const {
 	return to_string(originalOsmID);
 }
 
+// Check if there's a value for a given key
+bool OsmLuaProcessing::Holds(const string& key) const {
+	// NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawHolds
+	return currentPostScanTags->find(key)!=currentPostScanTags->end();
+}
+
+// Get an OSM tag for a given key (or return empty string if none)
+const string OsmLuaProcessing::Find(const string& key) const {
+	// NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawFind
+	auto it = currentPostScanTags->find(key);
+	if (it == currentPostScanTags->end()) return EMPTY_STRING;
+	return it->second;
+}
+
+// Check if an object has any tags
+bool OsmLuaProcessing::HasTags() const {
+	return isPostScanRelation ? !currentPostScanTags->empty() : !currentTags->empty();
+}
+
 // ----	Spatial queries called from Lua
 
 vector<string> OsmLuaProcessing::FindIntersecting(const string &layerName) {
@@ -771,6 +815,11 @@ std::vector<double> OsmLuaProcessing::Centroid(kaguya::VariadicArgType algorithm
 void OsmLuaProcessing::Accept() {
 	relationAccepted = true;
 }
+// Set a tag in post-scan phase
+void OsmLuaProcessing::SetTag(const std::string &key, const std::string &value) {
+	if (!isPostScanRelation) throw std::runtime_error("SetTag can only be used in relation_postscan_function");
+	osmStore.scannedRelations.set_relation_tag(originalOsmID, key, value);
+}
 
 void OsmLuaProcessing::removeAttributeIfNeeded(const string& key) {
 	// Does it exist?
@@ -882,7 +931,6 @@ void OsmLuaProcessing::setVectorLayerMetadata(const uint_least8_t layer, const s
 bool OsmLuaProcessing::scanRelation(WayID id, const TagMap& tags) {
 	reset();
 	originalOsmID = id;
-	isWay = false;
 	isRelation = true;
 	currentTags = &tags;
 	try {
@@ -899,11 +947,29 @@ bool OsmLuaProcessing::scanRelation(WayID id, const TagMap& tags) {
 	return true;
 }
 
+// Post-scan relations - typically used for bouncing down values from nested relations
+void OsmLuaProcessing::postScanRelations() {
+	if (!supportsPostScanRelations) return;
+
+	// Adjust the function pointers for tag-related functions
+	inPostScanRelations = true;
+	luaState["Holds"] = &rawHoldsPostScanRelations;
+	luaState["Find"] = &rawFindPostScanRelations;
+
+	for (const auto &relp : osmStore.scannedRelations.relationsForRelations) {
+		reset();
+		isPostScanRelation = true;
+		RelationID id = relp.first;
+		originalOsmID = id;
+		currentPostScanTags = &(osmStore.scannedRelations.relation_tags(id));
+		relationList = osmStore.scannedRelations.relations_for_relation_with_parents(id);
+		luaState["relation_postscan_function"](this);
+	}
+}
+
 bool OsmLuaProcessing::setNode(NodeID id, LatpLon node, const TagMap& tags) {
 	reset();
 	originalOsmID = id;
-	isWay = false;
-	isRelation = false;
 	lon = node.lon;
 	latp= node.latp;
 	currentTags = &tags;
@@ -939,7 +1005,6 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap
 	wayEmitted = false;
 	originalOsmID = wayId;
 	isWay = true;
-	isRelation = false;
 	llVecPtr = &llVec;
 	outerWayVecPtr = nullptr;
 	innerWayVecPtr = nullptr;
@@ -1004,6 +1069,10 @@ void OsmLuaProcessing::setRelation(
 	innerWayVecPtr = &innerWayVec;
 	currentTags = &tags;
 
+	if (supportsReadingRelations && osmStore.scannedRelations.relation_in_any_relations(originalOsmID)) {
+		relationList = osmStore.scannedRelations.relations_for_relation(originalOsmID);
+	}
+
 	// Start Lua processing for relation
 	if (!isNativeMP && !supportsWritingRelations) return;
 	try {

src/pbf_processor.cpp

@@ -221,6 +221,12 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG
 					std::string role(roleView.data(), roleView.size());
 					osmStore.scannedRelations.relation_contains_node(relid, lastID, role);
 				}
+			} else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::RELATION) {
+				if (isAccepted) {
+					const auto& roleView = pb.stringTable[pbfRelation.roles_sid[n]];
+					std::string role(roleView.data(), roleView.size());
+					osmStore.scannedRelations.relation_contains_relation(relid, lastID, role);
+				}
 			} else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::WAY) {
 				if (lastID >= pow(2,42)) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID));
 				osmStore.mark_way_used(static_cast<WayID>(lastID));
@@ -674,6 +680,10 @@ int PbfProcessor::ReadPbfFile(
 #endif
 		}
 
+		if(phase == ReadPhase::RelationScan) {
+			auto output = generate_output();
+			output->postScanRelations();
+		}
 		if(phase == ReadPhase::Nodes) {
 			osmStore.nodes.finalize(threadNum);
 			osmStore.usedNodes.clear();