GeoJSON as alternative to shapefiles (#630)

CMakeLists.txt

@@ -75,6 +75,7 @@ file(GLOB tilemaker_src_files
 	src/external/streamvbyte_decode.cc
 	src/external/streamvbyte_encode.cc
 	src/external/streamvbyte_zigzag.cc
+	src/geojson_processor.cpp
 	src/geom.cpp
 	src/helpers.cpp
 	src/mbtiles.cpp
@@ -89,11 +90,11 @@ file(GLOB tilemaker_src_files
 	src/pbf_reader.cpp
 	src/pmtiles.cpp
 	src/pooled_string.cpp
-	src/read_shp.cpp
 	src/sharded_node_store.cpp
 	src/sharded_way_store.cpp
 	src/shared_data.cpp
 	src/shp_mem_tiles.cpp
+	src/shp_processor.cpp
 	src/sorted_node_store.cpp
 	src/sorted_way_store.cpp
 	src/tile_data.cpp

Makefile

@@ -99,6 +99,7 @@ tilemaker: \
 	src/external/streamvbyte_decode.o \
 	src/external/streamvbyte_encode.o \
 	src/external/streamvbyte_zigzag.o \
+	src/geojson_processor.o \
 	src/geom.o \
 	src/helpers.o \
 	src/mbtiles.o \
@@ -113,11 +114,11 @@ tilemaker: \
 	src/pbf_reader.o \
 	src/pmtiles.o \
 	src/pooled_string.o \
-	src/read_shp.o \
 	src/sharded_node_store.o \
 	src/sharded_way_store.o \
 	src/shared_data.o \
 	src/shp_mem_tiles.o \
+	src/shp_processor.o \
 	src/sorted_node_store.o \
 	src/sorted_way_store.o \
 	src/tile_data.o \

include/geojson_processor.h

@@ -0,0 +1,46 @@
+/*! \file */ 
+#ifndef _GEOJSON_PROCESSOR_H
+#define _GEOJSON_PROCESSOR_H
+
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <map>
+#include "geom.h"
+#include "output_object.h"
+#include "osm_lua_processing.h"
+#include "attribute_store.h"
+
+class GeoJSONProcessor {
+
+public:
+	GeoJSONProcessor(Box &clippingBox, 
+	                 uint threadNum,
+	                 class ShpMemTiles &shpMemTiles,
+	                 OsmLuaProcessing &osmLuaProcessing) : 
+		 clippingBox(clippingBox), threadNum(threadNum),
+		 shpMemTiles(shpMemTiles), osmLuaProcessing(osmLuaProcessing)
+	{}
+
+	void read(class LayerDef &layer, uint layerNum);
+
+private:	
+	Box clippingBox;
+	unsigned threadNum;
+	ShpMemTiles &shpMemTiles;
+	OsmLuaProcessing &osmLuaProcessing;
+	std::mutex attributeMutex;
+
+	template <bool Flag, typename T>
+	void processFeature(rapidjson::GenericObject<Flag, T> feature, class LayerDef &layer, uint layerNum);
+
+	template <bool Flag, typename T>
+	Polygon polygonFromGeoJSONArray(const rapidjson::GenericArray<Flag, T> &coords);
+
+	template <bool Flag, typename T>
+	std::vector<Point> pointsFromGeoJSONArray(const rapidjson::GenericArray<Flag, T> &arr);
+
+	AttributeIndex readProperties(const rapidjson::Value &pr, bool &hasName, std::string &name, LayerDef &layer, unsigned &minzoom);
+};
+
+#endif //_GEOJSON_PROCESSOR_H

include/geojson.h → include/geojson_writer.h

@@ -1,14 +1,14 @@
 /*! \file */ 
-#ifndef _GEOJSON_H
-#define _GEOJSON_H
+#ifndef _GEOJSON_WRITER_H
+#define _GEOJSON_WRITER_H
 
 /*
 	GeoJSON writer for boost::geometry objects, using RapidJSON.
 	This isn't core tilemaker functionality but helps with debugging.
 	As yet it only outputs (Multi)Polygons but can be extended for more types.
 
 	Example:
-		auto gj = GeoJSON()
+		auto gj = GeoJSONWriter();
 		gj.addGeometry(myMultiPolygon);
 		gj.finalise();
 		std::cout << gj.toString() << std::endl;
@@ -32,11 +32,11 @@ typedef boost::variant<Point,Linestring,MultiLinestring,Polygon,MultiPolygon,Rin
 
 using namespace rapidjson;
 
-struct GeoJSON {
+struct GeoJSONWriter {
 	Document document;
 	std::vector<AnyGeometry> geometries;
 
-	GeoJSON() {
+	GeoJSONWriter() {
 		document.SetObject();
 		document.AddMember("type", Value().SetString("FeatureCollection"), document.GetAllocator());
 	}
@@ -134,4 +134,4 @@ struct GeoJSON {
 	}
 };
 
-#endif //_GEOJSON_H
+#endif //_GEOJSON_WRITER_H

include/shared_data.h

@@ -36,6 +36,10 @@ struct LayerDef {
 	std::string indexName;
 	std::map<std::string, uint> attributeMap;
 	bool writeTo;
+
+	const bool useColumn(std::string &col) {
+		return allSourceColumns || (std::find(sourceColumns.begin(), sourceColumns.end(), col) != sourceColumns.end());
+	}
 };
 
 ///\brief Defines layers used in map rendering

include/shp_mem_tiles.h

@@ -16,7 +16,7 @@ class ShpMemTiles : public TileDataSource
 	void CreateNamedLayerIndex(const std::string& layerName);
 
 	// Used in shape file loading
-	void StoreShapefileGeometry(
+	void StoreGeometry(
 		uint_least8_t layerNum,
 		const std::string& layerName, 
 		enum OutputGeometryType geomType,
@@ -64,6 +64,7 @@ class ShpMemTiles : public TileDataSource
 	std::vector<OutputObject> indexedGeometries;				// prepared boost::geometry objects (from shapefiles)
 	std::map<uint, std::string> indexedGeometryNames;			//  | optional names for each one
 	std::map<std::string, RTree> indices;			// Spatial indices, boost::geometry::index objects for shapefile indices
+	std::mutex indexMutex;
 };
 
 #endif //_OSM_MEM_TILES

include/shp_processor.h

@@ -0,0 +1,52 @@
+/*! \file */ 
+#ifndef _SHP_PROCESSOR_H
+#define _SHP_PROCESSOR_H
+
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <map>
+#include "geom.h"
+#include "output_object.h"
+#include "osm_lua_processing.h"
+#include "attribute_store.h"
+
+// Shapelib
+#include "shapefil.h"
+
+class ShpProcessor {
+
+public:
+	ShpProcessor(Box &clippingBox, 
+	             uint threadNum,
+	             class ShpMemTiles &shpMemTiles,
+	             OsmLuaProcessing &osmLuaProcessing) : 
+		 clippingBox(clippingBox), threadNum(threadNum),
+		 shpMemTiles(shpMemTiles), osmLuaProcessing(osmLuaProcessing)
+	{}
+
+	// Read shapefile, and create OutputObjects for all objects within the specified bounding box
+	void read(class LayerDef &layer, uint layerNum);
+
+private:	
+	Box clippingBox;
+	unsigned threadNum;
+	ShpMemTiles &shpMemTiles;
+	OsmLuaProcessing &osmLuaProcessing;
+	std::mutex attributeMutex;
+
+	void fillPointArrayFromShapefile(std::vector<Point> *points, SHPObject *shape, uint part);
+
+	// Read requested attributes from a shapefile, and encode into an OutputObject
+	AttributeIndex readShapefileAttributes(DBFHandle &dbf, int recordNum, 
+	                                       std::unordered_map<int,std::string> &columnMap,
+	                                       std::unordered_map<int,int> &columnTypeMap,
+	                                       LayerDef &layer, uint &minzoom);
+
+	// Process an individual shapefile record
+	void processShapeGeometry(SHPObject* shape, AttributeIndex attrIdx, 
+	                          const LayerDef &layer, uint layerNum, bool hasName, const std::string &name);
+};
+
+#endif //_SHP_PROCESSOR_H
+