From ad86ab4a011e46122ca958eeb92e6d5b7a6398db Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Mon, 1 Apr 2024 12:24:47 -0400 Subject: [PATCH] remove FAT_TILE_INDEX (#700) --- CMakeLists.txt | 1 + Makefile | 9 ++- docs/INSTALL.md | 8 --- include/coordinates.h | 9 --- include/osm_mem_tiles.h | 2 +- include/shp_mem_tiles.h | 2 +- include/tile_coordinates_set.h | 47 +++++++++++++ include/tile_data.h | 102 ++++++++++++++++------------- src/osm_mem_tiles.cpp | 4 +- src/shared_data.cpp | 6 -- src/shp_mem_tiles.cpp | 13 ++-- src/tile_coordinates_set.cpp | 57 ++++++++++++++++ src/tile_data.cpp | 101 +++++++++++++--------------- src/tilemaker.cpp | 24 +++++-- test/tile_coordinates_set.test.cpp | 58 ++++++++++++++++ 15 files changed, 298 insertions(+), 145 deletions(-) create mode 100644 include/tile_coordinates_set.h create mode 100644 src/tile_coordinates_set.cpp create mode 100644 test/tile_coordinates_set.test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ff4b32e..9320bef8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,7 @@ file(GLOB tilemaker_src_files src/sorted_node_store.cpp src/sorted_way_store.cpp src/tag_map.cpp + src/tile_coordinates_set.cpp src/tile_data.cpp src/tilemaker.cpp src/tile_worker.cpp diff --git a/Makefile b/Makefile index 174e9d3a..fea883e0 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ tilemaker: \ src/sorted_node_store.o \ src/sorted_way_store.o \ src/tag_map.o \ + src/tile_coordinates_set.o \ src/tile_data.o \ src/tilemaker.o \ src/tile_worker.o \ @@ -130,7 +131,8 @@ test: \ test_relation_roles \ test_significant_tags \ test_sorted_node_store \ - test_sorted_way_store + test_sorted_way_store \ + test_tile_coordinates_set test_append_vector: \ src/mmap_allocator.o \ @@ -193,6 +195,11 @@ test_sorted_way_store: \ test/sorted_way_store.test.o $(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store +test_tile_coordinates_set: \ + src/tile_coordinates_set.o \ + test/tile_coordinates_set.test.o + $(CXX) $(CXXFLAGS) -o test.tile_coordinates_set $^ $(INC) $(LIB) $(LDFLAGS) && ./test.tile_coordinates_set + test_pbf_reader: \ src/helpers.o \ src/pbf_reader.o \ diff --git a/docs/INSTALL.md b/docs/INSTALL.md index e2ca6633..26cef84c 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -65,11 +65,3 @@ The docker container can be run like this: docker run -v /Users/Local/Downloads/:/srv -i -t --rm tilemaker /srv/germany-latest.osm.pbf --output=/srv/germany.mbtiles Keep in mind to map the volume your .osm.pbf files are in to a path within your docker container, as seen in the example above. - -### Compile-time options - -tilemaker has a compile-time option that increases memory usage but may be useful in certain circumstances. You can include it when building like this: - - make "CONFIG=-DFAT_TILE_INDEX" - -FAT_TILE_INDEX allows you to generate vector tiles at zoom level 17 or greater. You almost certainly don't need to do this. Vector tiles are usually generated up to zoom 14 (sometimes 15), and then the browser/app client uses the vector data to scale up at subsequent zoom levels. diff --git a/include/coordinates.h b/include/coordinates.h index 6d34ca61..1ff107b2 100644 --- a/include/coordinates.h +++ b/include/coordinates.h @@ -24,17 +24,8 @@ typedef uint64_t RelationID; typedef std::vector WayVec; -#ifdef FAT_TILE_INDEX // Supports up to z22 typedef uint32_t TileCoordinate; -typedef uint16_t Z6Offset; -#define TILE_COORDINATE_MAX UINT32_MAX -#else -// Supports up to z14 -typedef uint16_t TileCoordinate; -typedef uint8_t Z6Offset; -#define TILE_COORDINATE_MAX UINT16_MAX -#endif class TileCoordinates_ { diff --git a/include/osm_mem_tiles.h b/include/osm_mem_tiles.h index 3c920b08..a01b1cd9 100644 --- a/include/osm_mem_tiles.h +++ b/include/osm_mem_tiles.h @@ -31,7 +31,7 @@ class OsmMemTiles : public TileDataSource { public: OsmMemTiles( size_t threadNum, - uint baseZoom, + uint indexZoom, bool includeID, const NodeStore& nodeStore, const WayStore& wayStore diff --git a/include/shp_mem_tiles.h b/include/shp_mem_tiles.h index 33d6e7c2..93d6334d 100644 --- a/include/shp_mem_tiles.h +++ b/include/shp_mem_tiles.h @@ -9,7 +9,7 @@ extern bool verbose; class ShpMemTiles : public TileDataSource { public: - ShpMemTiles(size_t threadNum, uint baseZoom); + ShpMemTiles(size_t threadNum, uint indexZoom); std::string name() const override { return "shp"; } diff --git a/include/tile_coordinates_set.h b/include/tile_coordinates_set.h new file mode 100644 index 00000000..acc7a45b --- /dev/null +++ b/include/tile_coordinates_set.h @@ -0,0 +1,47 @@ +#ifndef TILE_COORDINATES_SET_H +#define TILE_COORDINATES_SET_H + +#include +#include "coordinates.h" + +// Interface representing a bitmap of tiles of interest at a given zoom. +class TileCoordinatesSet { +public: + virtual bool test(TileCoordinate x, TileCoordinate y) const = 0; + virtual void set(TileCoordinate x, TileCoordinate y) = 0; + virtual size_t size() const = 0; + virtual size_t zoom() const = 0; +}; + +// Read-write implementation for precise sets; maximum zoom is +// generally expected to be z14. +class PreciseTileCoordinatesSet : public TileCoordinatesSet { +public: + PreciseTileCoordinatesSet(unsigned int zoom); + bool test(TileCoordinate x, TileCoordinate y) const override; + size_t size() const override; + size_t zoom() const override; + void set(TileCoordinate x, TileCoordinate y) override; + +private: + unsigned int zoom_; + std::vector tiles; +}; + +// Read-only implementation for a lossy set. Used when zoom is +// z15 or higher, extrapolates a result based on a set for a lower zoom. +class LossyTileCoordinatesSet : public TileCoordinatesSet { +public: + LossyTileCoordinatesSet(unsigned int zoom, const TileCoordinatesSet& precise); + bool test(TileCoordinate x, TileCoordinate y) const override; + size_t size() const override; + size_t zoom() const override; + void set(TileCoordinate x, TileCoordinate y) override; + +private: + unsigned int zoom_; + const TileCoordinatesSet& tiles; + unsigned int scale; +}; + +#endif diff --git a/include/tile_data.h b/include/tile_data.h index b78463e2..32e892d3 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -11,6 +11,7 @@ #include "append_vector.h" #include "clip_cache.h" #include "mmap_allocator.h" +#include "tile_coordinates_set.h" #define TILE_DATA_ID_SIZE 34 @@ -23,24 +24,27 @@ class TileBbox; #define CLUSTER_ZOOM_WIDTH (1 << CLUSTER_ZOOM) #define CLUSTER_ZOOM_AREA (CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH) +// TileDataSource indexes which tiles have objects in them. The indexed zoom +// is at most z14; we'll clamp to z14 if the base zoom is higher than z14. +// +// As a result, we need at most 15 bits to store an X/Y coordinate. For efficiency, +// we bucket the world into 4,096 z6 tiles, which each contain some number of +// z14 objects. This lets us use only 8 bits to store an X/Y coordinate. +// +// Because index zoom is lower than base zoom in the case where base zoom is +// z15+, we'll get false positives when looking up objects in the index, +// since, e.g., a single z14 tile covers 4 z15 tiles. +// +// This is OK: when writing the z15 tile, there's a clipping step that will filter +// out the false positives. +typedef uint8_t Z6Offset; + struct OutputObjectXY { OutputObject oo; Z6Offset x; Z6Offset y; }; -class TileCoordinatesSet { -public: - TileCoordinatesSet(uint zoom); - bool test(TileCoordinate x, TileCoordinate y) const; - void set(TileCoordinate x, TileCoordinate y); - size_t size() const; - -private: - uint zoom; - std::vector tiles; -}; - struct OutputObjectXYID { OutputObject oo; Z6Offset x; @@ -51,12 +55,12 @@ struct OutputObjectXYID { template void finalizeObjects( const std::string& name, const size_t& threadNum, - const unsigned int& baseZoom, + const unsigned int& indexZoom, typename std::vector>::iterator begin, typename std::vector>::iterator end, typename std::vector>& lowZoom ) { - size_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1; + size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; #ifdef CLOCK_MONOTONIC timespec startTs, endTs; clock_gettime(CLOCK_MONOTONIC, &startTs); @@ -105,7 +109,7 @@ template void finalizeObjects( boost::sort::block_indirect_sort( it->begin(), it->end(), - [baseZoom](const OO& a, const OO& b) { + [indexZoom](const OO& a, const OO& b) { // Cluster by parent zoom, so that a subsequent search // can find a contiguous range of entries for any tile // at zoom 6 or higher. @@ -113,14 +117,14 @@ template void finalizeObjects( const size_t aY = a.y; const size_t bX = b.x; const size_t bY = b.y; - for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) { - const auto aXz = aX / (1 << (baseZoom - z)); - const auto bXz = bX / (1 << (baseZoom - z)); + for (size_t z = CLUSTER_ZOOM; z <= indexZoom; z++) { + const auto aXz = aX / (1 << (indexZoom - z)); + const auto bXz = bX / (1 << (indexZoom - z)); if (aXz != bXz) return aXz < bXz; - const auto aYz = aY / (1 << (baseZoom - z)); - const auto bYz = bY / (1 << (baseZoom - z)); + const auto aYz = aY / (1 << (indexZoom - z)); + const auto bYz = bY / (1 << (indexZoom - z)); if (aYz != bYz) return aYz < bYz; @@ -136,13 +140,13 @@ template void finalizeObjects( } template void collectTilesWithObjectsAtZoomTemplate( - const unsigned int& baseZoom, + const unsigned int& indexZoom, const typename std::vector>::iterator objects, const size_t size, - std::vector& zooms + std::vector>& zooms ) { size_t maxZoom = zooms.size() - 1; - uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1; + uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; int64_t lastX = -1; int64_t lastY = -1; for (size_t i = 0; i < size; i++) { @@ -155,15 +159,15 @@ template void collectTilesWithObjectsAtZoomTemplate( TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (baseZoom - maxZoom)); - TileCoordinate y = baseY / (1 << (baseZoom - maxZoom)); + TileCoordinate x = baseX / (1 << (indexZoom - maxZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - maxZoom)); if (lastX != x || lastY != y) { lastX = x; lastY = y; for (int zoom = maxZoom; zoom >= 0; zoom--) { - zooms[zoom].set(x, y); + zooms[zoom]->set(x, y); x /= 2; y /= 2; } @@ -183,7 +187,7 @@ inline OutputObjectID outputObjectWithId(const OutputObjectXYI } template void collectLowZoomObjectsForTile( - const unsigned int& baseZoom, + const unsigned int& indexZoom, typename std::vector> objects, unsigned int zoom, const TileCoordinates& dstIndex, @@ -192,7 +196,7 @@ template void collectLowZoomObjectsForTile( if (zoom >= CLUSTER_ZOOM) throw std::runtime_error("collectLowZoomObjectsForTile should not be called for high zooms"); - uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1; + uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; for (size_t i = 0; i < objects.size(); i++) { const size_t z6x = i / CLUSTER_ZOOM_WIDTH; @@ -204,8 +208,8 @@ template void collectLowZoomObjectsForTile( TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y; // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (baseZoom - zoom)); - TileCoordinate y = baseY / (1 << (baseZoom - zoom)); + TileCoordinate x = baseX / (1 << (indexZoom - zoom)); + TileCoordinate y = baseY / (1 << (indexZoom - zoom)); if (dstIndex.x == x && dstIndex.y == y) { if (objects[i][j].oo.minZoom <= zoom) { @@ -217,28 +221,36 @@ template void collectLowZoomObjectsForTile( } template void collectObjectsForTileTemplate( - const unsigned int& baseZoom, + const unsigned int& indexZoom, typename std::vector>::iterator objects, size_t iStart, size_t iEnd, unsigned int zoom, - const TileCoordinates& dstIndex, + TileCoordinates dstIndex, std::vector& output ) { if (zoom < CLUSTER_ZOOM) throw std::runtime_error("collectObjectsForTileTemplate should not be called for low zooms"); - uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1; + // When base zoom is z15 or higher, we need to scale down to z14. + unsigned int clampedZoom = zoom; + while(clampedZoom > indexZoom) { + clampedZoom--; + dstIndex.x /= 2; + dstIndex.y /= 2; + } + + uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1; for (size_t i = iStart; i < iEnd; i++) { // If z >= 6, we can compute the exact bounds within the objects array. // Translate to the base zoom, then do a binary search to find // the starting point. - TileCoordinate z6x = dstIndex.x / (1 << (zoom - CLUSTER_ZOOM)); - TileCoordinate z6y = dstIndex.y / (1 << (zoom - CLUSTER_ZOOM)); + TileCoordinate z6x = dstIndex.x / (1 << (clampedZoom - CLUSTER_ZOOM)); + TileCoordinate z6y = dstIndex.y / (1 << (clampedZoom - CLUSTER_ZOOM)); - TileCoordinate baseX = dstIndex.x * (1 << (baseZoom - zoom)); - TileCoordinate baseY = dstIndex.y * (1 << (baseZoom - zoom)); + TileCoordinate baseX = dstIndex.x * (1 << (indexZoom - clampedZoom)); + TileCoordinate baseY = dstIndex.y * (1 << (indexZoom - clampedZoom)); Z6Offset needleX = baseX - z6x * z6OffsetDivisor; Z6Offset needleY = baseY - z6y * z6OffsetDivisor; @@ -247,7 +259,7 @@ template void collectObjectsForTileTemplate( // into two arrays, one of x/y and one of OOs. Would have better locality for // searching, too. OutputObject dummyOo(POINT_, 0, 0, 0, 0); - const size_t bz = baseZoom; + const size_t bz = indexZoom; const OO targetXY = {dummyOo, needleX, needleY }; auto iter = std::lower_bound( @@ -284,8 +296,8 @@ template void collectObjectsForTileTemplate( TileCoordinate baseY = z6y * z6OffsetDivisor + iter->y; // Translate the x, y at the requested zoom level - TileCoordinate x = baseX / (1 << (baseZoom - zoom)); - TileCoordinate y = baseY / (1 << (baseZoom - zoom)); + TileCoordinate x = baseX / (1 << (indexZoom - clampedZoom)); + TileCoordinate y = baseY / (1 << (indexZoom - clampedZoom)); if (dstIndex.x == x && dstIndex.y == y) { if (iter->oo.minZoom <= zoom) { @@ -354,7 +366,7 @@ class TileDataSource { boost::geometry::index::rtree< std::pair, oo_rtree_param_type> boxRtree; boost::geometry::index::rtree< std::pair, oo_rtree_param_type> boxRtreeWithIds; - unsigned int baseZoom; + unsigned int indexZoom; std::vector pointStores; std::vector linestringStores; @@ -367,11 +379,11 @@ class TileDataSource { std::deque>> pendingSmallIndexObjects; public: - TileDataSource(size_t threadNum, unsigned int baseZoom, bool includeID); + TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID); - void collectTilesWithObjectsAtZoom(std::vector& zooms); + void collectTilesWithObjectsAtZoom(std::vector>& zooms); - void collectTilesWithLargeObjectsAtZoom(std::vector& zooms); + void collectTilesWithLargeObjectsAtZoom(std::vector>& zooms); void collectObjectsForTile(uint zoom, TileCoordinates dstIndex, std::vector& output); void finalize(size_t threadNum); @@ -486,7 +498,7 @@ class TileDataSource { void populateTilesAtZoom( const std::vector& sources, - std::vector& zooms + std::vector>& zooms ); #endif //_TILE_DATA_H diff --git a/src/osm_mem_tiles.cpp b/src/osm_mem_tiles.cpp index 7dc03f45..bc038767 100644 --- a/src/osm_mem_tiles.cpp +++ b/src/osm_mem_tiles.cpp @@ -7,12 +7,12 @@ thread_local GeometryCache linestringCache; OsmMemTiles::OsmMemTiles( size_t threadNum, - uint baseZoom, + uint indexZoom, bool includeID, const NodeStore& nodeStore, const WayStore& wayStore ) - : TileDataSource(threadNum, baseZoom, includeID), + : TileDataSource(threadNum, indexZoom, includeID), nodeStore(nodeStore), wayStore(wayStore) { diff --git a/src/shared_data.cpp b/src/shared_data.cpp index da9787d8..f68f2f14 100644 --- a/src/shared_data.cpp +++ b/src/shared_data.cpp @@ -259,12 +259,6 @@ void Config::readConfig(rapidjson::Document &jsonConfig, bool &hasClippingBox, B cout << "tilemaker may have excessive memory, time, and space requirements at higher zooms. You can find more information in the docs/ folder." << endl; cout << "**** WARNING ****" << endl; } -#ifndef FAT_TILE_INDEX - if (endZoom>16) { - cerr << "Compile tilemaker with -DFAT_TILE_INDEX to enable tile output at zoom level 17 or greater" << endl; - exit (EXIT_FAILURE); - } -#endif compressOpt = jsonConfig["settings"]["compress"].GetString(); combineBelow = jsonConfig["settings"].HasMember("combine_below") ? jsonConfig["settings"]["combine_below"].GetUint() : 0; diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index 05383d14..6fa01377 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -6,8 +6,8 @@ using namespace std; namespace geom = boost::geometry; extern bool verbose; -ShpMemTiles::ShpMemTiles(size_t threadNum, uint baseZoom) - : TileDataSource(threadNum, baseZoom, false) +ShpMemTiles::ShpMemTiles(size_t threadNum, uint indexZoom) + : TileDataSource(threadNum, indexZoom, false) { } // Look for shapefile objects that fulfil a spatial query (e.g. intersects) @@ -57,7 +57,6 @@ void ShpMemTiles::CreateNamedLayerIndex(const std::string& layerName) { indices[layerName]=RTree(); bitIndices[layerName] = std::vector(); - const uint8_t indexZoom = std::min(baseZoom, 14u); bitIndices[layerName].resize((1 << indexZoom) * (1 << indexZoom)); } @@ -65,7 +64,6 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con // Check if any tiles in the bitmap might intersect this shape. // If none, downstream code can skip querying the r-tree. auto& bitvec = bitIndices.at(layerName); - const uint8_t indexZoom = std::min(baseZoom, 14u); double lon1 = box.min_corner().x(); double lat1 = box.min_corner().y(); @@ -112,8 +110,8 @@ void ShpMemTiles::StoreGeometry( Point sp(p->x()*10000000.0, p->y()*10000000.0); NodeID oid = storePoint(sp); oo = std::make_shared(geomType, layerNum, oid, attrIdx, minzoom); - tilex = lon2tilex(p->x(), baseZoom); - tiley = latp2tiley(p->y(), baseZoom); + tilex = lon2tilex(p->x(), indexZoom); + tiley = latp2tiley(p->y(), indexZoom); addObjectToSmallIndex(TileCoordinates(tilex, tiley), *oo, 0); } else { return; } } break; @@ -157,10 +155,9 @@ void ShpMemTiles::StoreGeometry( if (hasName) { indexedGeometryNames[id] = name; } indexedGeometries.push_back(*oo); - // Store a bitmap of which tiles at the basezoom might intersect + // Store a bitmap of which tiles at the indexZoom might intersect // this shape. auto& bitvec = bitIndices.at(layerName); - const uint8_t indexZoom = std::min(baseZoom, 14u); double lon1 = box.min_corner().x(); double lat1 = box.min_corner().y(); double lon2 = box.max_corner().x(); diff --git a/src/tile_coordinates_set.cpp b/src/tile_coordinates_set.cpp new file mode 100644 index 00000000..2f63e468 --- /dev/null +++ b/src/tile_coordinates_set.cpp @@ -0,0 +1,57 @@ +#include "tile_coordinates_set.h" +#include +#include + +PreciseTileCoordinatesSet::PreciseTileCoordinatesSet(unsigned int zoom): + zoom_(zoom), + tiles((1 << zoom) * (1 << zoom)) {} + +bool PreciseTileCoordinatesSet::test(TileCoordinate x, TileCoordinate y) const { + uint64_t loc = x * (1 << zoom_) + y; + if (loc >= tiles.size()) + return false; + + return tiles[loc]; +} + +size_t PreciseTileCoordinatesSet::zoom() const { + return zoom_; +} + +size_t PreciseTileCoordinatesSet::size() const { + size_t rv = 0; + for (int i = 0; i < tiles.size(); i++) + if (tiles[i]) + rv++; + + return rv; +} + +void PreciseTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { + uint64_t loc = x * (1 << zoom_) + y; + if (loc >= tiles.size()) + return; + tiles[loc] = true; +} + +LossyTileCoordinatesSet::LossyTileCoordinatesSet(unsigned int zoom, const TileCoordinatesSet& underlying) : zoom_(zoom), tiles(underlying), scale(1 << (zoom - underlying.zoom())) { + if (zoom <= underlying.zoom()) + throw std::out_of_range("LossyTileCoordinatesSet: zoom (" + std::to_string(zoom_) + ") must be greater than underlying set's zoom (" + std::to_string(underlying.zoom()) + ")"); +} + +bool LossyTileCoordinatesSet::test(TileCoordinate x, TileCoordinate y) const { + return tiles.test(x / scale, y / scale); +} + +size_t LossyTileCoordinatesSet::size() const { + return tiles.size() * scale * scale; +} + +size_t LossyTileCoordinatesSet::zoom() const { + return zoom_; +} + +void LossyTileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { + throw std::logic_error("LossyTileCoordinatesSet::set() is not implemented; LossyTileCoordinatesSet is read-only"); +} + diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 407f534a..db9ffaac 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -8,56 +8,34 @@ using namespace std; extern bool verbose; -TileCoordinatesSet::TileCoordinatesSet(uint zoom): - zoom(zoom), - tiles((1 << zoom) * (1 << zoom)) {} - -bool TileCoordinatesSet::test(TileCoordinate x, TileCoordinate y) const { - uint64_t loc = x * (1 << zoom) + y; - if (loc >= tiles.size()) - return false; - - return tiles[loc]; -} - -size_t TileCoordinatesSet::size() const { - size_t rv = 0; - for (int i = 0; i < tiles.size(); i++) - if (tiles[i]) - rv++; - - return rv; -} - -void TileCoordinatesSet::set(TileCoordinate x, TileCoordinate y) { - uint64_t loc = x * (1 << zoom) + y; - if (loc >= tiles.size()) - return; - tiles[loc] = true; -} - thread_local LeasedStore pointStore; thread_local LeasedStore linestringStore; thread_local LeasedStore multilinestringStore; thread_local LeasedStore multipolygonStore; -TileDataSource::TileDataSource(size_t threadNum, unsigned int baseZoom, bool includeID) +TileDataSource::TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID) : includeID(includeID), - z6OffsetDivisor(baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1), + z6OffsetDivisor(indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1), objectsMutex(threadNum * 4), objects(CLUSTER_ZOOM_AREA), lowZoomObjects(CLUSTER_ZOOM_AREA), objectsWithIds(CLUSTER_ZOOM_AREA), lowZoomObjectsWithIds(CLUSTER_ZOOM_AREA), - baseZoom(baseZoom), + indexZoom(indexZoom), pointStores(threadNum), linestringStores(threadNum), multilinestringStores(threadNum), multipolygonStores(threadNum), - multiPolygonClipCache(ClipCache(threadNum, baseZoom)), - multiLinestringClipCache(ClipCache(threadNum, baseZoom)) + multiPolygonClipCache(ClipCache(threadNum, indexZoom)), + multiLinestringClipCache(ClipCache(threadNum, indexZoom)) { + // TileDataSource can only index up to zoom 14. The caller is responsible for + // ensuring it does not use a higher zoom. + if (indexZoom > 14) + throw std::out_of_range("TileDataSource: indexZoom cannot be higher than 14, but was " + std::to_string(indexZoom)); + + shardBits = 0; numShards = 1; while(numShards < threadNum) { @@ -86,8 +64,8 @@ void TileDataSource::finalize(size_t threadNum) { std::cout << "indexed " << finalized << " contended objects" << std::endl; - finalizeObjects(name(), threadNum, baseZoom, objects.begin(), objects.end(), lowZoomObjects); - finalizeObjects(name(), threadNum, baseZoom, objectsWithIds.begin(), objectsWithIds.end(), lowZoomObjectsWithIds); + finalizeObjects(name(), threadNum, indexZoom, objects.begin(), objects.end(), lowZoomObjects); + finalizeObjects(name(), threadNum, indexZoom, objectsWithIds.begin(), objectsWithIds.end(), lowZoomObjectsWithIds); } void TileDataSource::addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id) { @@ -96,7 +74,7 @@ void TileDataSource::addObjectToSmallIndex(const TileCoordinates& index, const O const size_t z6y = index.y / z6OffsetDivisor; if (z6x >= 64 || z6y >= 64) { - if (verbose) std::cerr << "ignoring OutputObject with invalid z" << baseZoom << " coordinates " << index.x << ", " << index.y << " (id: " << id << ")" << std::endl; + if (verbose) std::cerr << "ignoring OutputObject with invalid z" << indexZoom << " coordinates " << index.x << ", " << index.y << " (id: " << id << ")" << std::endl; return; } @@ -139,15 +117,16 @@ void TileDataSource::addObjectToSmallIndexUnsafe(const TileCoordinates& index, c }); } -void TileDataSource::collectTilesWithObjectsAtZoom(std::vector& zooms) { +void TileDataSource::collectTilesWithObjectsAtZoom(std::vector>& zooms) { // Scan through all shards. Convert to base zoom, then convert to the requested zoom. - collectTilesWithObjectsAtZoomTemplate(baseZoom, objects.begin(), objects.size(), zooms); - collectTilesWithObjectsAtZoomTemplate(baseZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); + collectTilesWithObjectsAtZoomTemplate(indexZoom, objects.begin(), objects.size(), zooms); + collectTilesWithObjectsAtZoomTemplate(indexZoom, objectsWithIds.begin(), objectsWithIds.size(), zooms); } -void addCoveredTilesToOutput(const uint baseZoom, std::vector& zooms, const Box& box) { +void addCoveredTilesToOutput(const uint indexZoom, std::vector>& zooms, const Box& box) { size_t maxZoom = zooms.size() - 1; - int scale = pow(2, baseZoom - maxZoom); +// std::cout << "addCoveredTilesToOutput maxZoom=" << maxZoom << ", indexZoom - maxZoom = " << (indexZoom - maxZoom) << std::endl; + int scale = pow(2, indexZoom - maxZoom); TileCoordinate minx = box.min_corner().x() / scale; TileCoordinate maxx = box.max_corner().x() / scale; TileCoordinate miny = box.min_corner().y() / scale; @@ -157,7 +136,7 @@ void addCoveredTilesToOutput(const uint baseZoom, std::vector= 0; zoom--) { - zooms[zoom].set(zx, zy); + zooms[zoom]->set(zx, zy); zx /= 2; zy /= 2; } @@ -166,12 +145,12 @@ void addCoveredTilesToOutput(const uint baseZoom, std::vector& zooms) { +void TileDataSource::collectTilesWithLargeObjectsAtZoom(std::vector>& zooms) { for(auto const &result: boxRtree) - addCoveredTilesToOutput(baseZoom, zooms, result.first); + addCoveredTilesToOutput(indexZoom, zooms, result.first); for(auto const &result: boxRtreeWithIds) - addCoveredTilesToOutput(baseZoom, zooms, result.first); + addCoveredTilesToOutput(indexZoom, zooms, result.first); } // Copy objects from the tile at dstIndex (in the dataset srcTiles) into output @@ -181,8 +160,8 @@ void TileDataSource::collectObjectsForTile( std::vector& output ) { if (zoom < CLUSTER_ZOOM) { - collectLowZoomObjectsForTile(baseZoom, lowZoomObjects, zoom, dstIndex, output); - collectLowZoomObjectsForTile(baseZoom, lowZoomObjectsWithIds, zoom, dstIndex, output); + collectLowZoomObjectsForTile(indexZoom, lowZoomObjects, zoom, dstIndex, output); + collectLowZoomObjectsForTile(indexZoom, lowZoomObjectsWithIds, zoom, dstIndex, output); return; } @@ -190,7 +169,6 @@ void TileDataSource::collectObjectsForTile( size_t iEnd = objects.size(); if (zoom >= CLUSTER_ZOOM) { - // Compute the x, y at the base zoom level TileCoordinate z6x = dstIndex.x / (1 << (zoom - CLUSTER_ZOOM)); TileCoordinate z6y = dstIndex.y / (1 << (zoom - CLUSTER_ZOOM)); @@ -202,8 +180,8 @@ void TileDataSource::collectObjectsForTile( iEnd = iStart + 1; } - collectObjectsForTileTemplate(baseZoom, objects.begin(), iStart, iEnd, zoom, dstIndex, output); - collectObjectsForTileTemplate(baseZoom, objectsWithIds.begin(), iStart, iEnd, zoom, dstIndex, output); + collectObjectsForTileTemplate(indexZoom, objects.begin(), iStart, iEnd, zoom, dstIndex, output); + collectObjectsForTileTemplate(indexZoom, objectsWithIds.begin(), iStart, iEnd, zoom, dstIndex, output); } // Copy objects from the large index into output @@ -212,7 +190,13 @@ void TileDataSource::collectLargeObjectsForTile( TileCoordinates dstIndex, std::vector& output ) { - int scale = pow(2, baseZoom - zoom); + unsigned int clampedZoom = zoom; + while (clampedZoom > indexZoom) { + clampedZoom--; + dstIndex.x /= 2; + dstIndex.y /= 2; + } + int scale = pow(2, indexZoom - clampedZoom); TileCoordinates srcIndex1( dstIndex.x *scale , dstIndex.y *scale ); TileCoordinates srcIndex2((dstIndex.x+1)*scale-1, (dstIndex.y+1)*scale-1); Box box = Box(geom::make(srcIndex1.x, srcIndex1.y), @@ -399,8 +383,11 @@ void TileDataSource::reportSize() const { void populateTilesAtZoom( const std::vector& sources, - std::vector& zooms + std::vector>& zooms ) { + if (zooms.size() > 15) + throw std::out_of_range("populateTilesAtZoom: expected at most z14 zooms (15), but found " + std::to_string(zooms.size()) + " vectors"); + for(size_t i=0; icollectTilesWithObjectsAtZoom(zooms); sources[i]->collectTilesWithLargeObjectsAtZoom(zooms); @@ -446,10 +433,10 @@ void TileDataSource::addGeometryToIndex( ) { unordered_set tileSet; try { - insertIntermediateTiles(geom, baseZoom, tileSet); + insertIntermediateTiles(geom, indexZoom, tileSet); bool polygonExists = false; - TileCoordinate minTileX = TILE_COORDINATE_MAX, maxTileX = 0, minTileY = TILE_COORDINATE_MAX, maxTileY = 0; + TileCoordinate minTileX = std::numeric_limits::max(), maxTileX = 0, minTileY = std::numeric_limits::max(), maxTileY = 0; for (auto it = tileSet.begin(); it != tileSet.end(); ++it) { TileCoordinates index = *it; minTileX = std::min(index.x, minTileX); @@ -498,7 +485,7 @@ void TileDataSource::addGeometryToIndex( ) { for (Linestring ls : geom) { unordered_set tileSet; - insertIntermediateTiles(ls, baseZoom, tileSet); + insertIntermediateTiles(ls, indexZoom, tileSet); for (auto it = tileSet.begin(); it != tileSet.end(); ++it) { TileCoordinates index = *it; for (const auto& output : outputs) { @@ -517,7 +504,7 @@ void TileDataSource::addGeometryToIndex( bool singleOuter = geom.size()==1; for (Polygon poly : geom) { unordered_set tileSetTmp; - insertIntermediateTiles(poly.outer(), baseZoom, tileSetTmp); + insertIntermediateTiles(poly.outer(), indexZoom, tileSetTmp); fillCoveredTiles(tileSetTmp); if (singleOuter) { tileSet = std::move(tileSetTmp); @@ -526,7 +513,7 @@ void TileDataSource::addGeometryToIndex( } } - TileCoordinate minTileX = TILE_COORDINATE_MAX, maxTileX = 0, minTileY = TILE_COORDINATE_MAX, maxTileY = 0; + TileCoordinate minTileX = std::numeric_limits::max(), maxTileX = 0, minTileY = std::numeric_limits::max(), maxTileY = 0; for (auto it = tileSet.begin(); it != tileSet.end(); ++it) { TileCoordinates index = *it; minTileX = std::min(index.x, minTileX); diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 9cda33f2..37fc9812 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -225,8 +225,10 @@ int main(const int argc, const char* argv[]) { AttributeStore attributeStore; class LayerDefinition layers(config.layers); - class OsmMemTiles osmMemTiles(options.threadNum, config.baseZoom, config.includeID, *nodeStore, *wayStore); - class ShpMemTiles shpMemTiles(options.threadNum, config.baseZoom); + + const unsigned int indexZoom = std::min(config.baseZoom, 14u); + class OsmMemTiles osmMemTiles(options.threadNum, indexZoom, config.includeID, *nodeStore, *wayStore); + class ShpMemTiles shpMemTiles(options.threadNum, indexZoom); osmMemTiles.open(); shpMemTiles.open(); @@ -332,7 +334,7 @@ int main(const int argc, const char* argv[]) { // The clipping bbox check is expensive - as an optimization, compute the set of // z6 tiles that are wholly covered by the clipping box. Membership in this // set is quick to test. - TileCoordinatesSet coveredZ6Tiles(6); + PreciseTileCoordinatesSet coveredZ6Tiles(6); if (hasClippingBox) { for (int x = 0; x < 1 << 6; x++) { for (int y = 0; y < 1 << 6; y++) { @@ -352,9 +354,12 @@ int main(const int argc, const char* argv[]) { } std::deque> tileCoordinates; - std::vector zoomResults; - for (uint zoom = 0; zoom <= sharedData.config.endZoom; zoom++) { - zoomResults.push_back(TileCoordinatesSet(zoom)); + std::vector> zoomResults; + zoomResults.reserve(sharedData.config.endZoom + 1); + + // Add PreciseTileCoordinatesSet, but only up to z14. + for (uint zoom = 0; zoom <= std::min(14u, sharedData.config.endZoom); zoom++) { + zoomResults.emplace_back(std::make_shared(zoom)); } { @@ -371,6 +376,11 @@ int main(const int argc, const char* argv[]) { #endif } + // Add LossyTileCoordinatesSet, if needed + for (uint zoom = 15u; zoom <= sharedData.config.endZoom; zoom++) { + zoomResults.emplace_back(std::make_shared(zoom, *zoomResults[14])); + } + std::cout << ", filtering tiles:" << std::flush; for (uint zoom=sharedData.config.startZoom; zoom <= sharedData.config.endZoom; zoom++) { std::cout << " z" << std::to_string(zoom) << std::flush; @@ -383,7 +393,7 @@ int main(const int argc, const char* argv[]) { int numTiles = 0; for (int x = 0; x < 1 << zoom; x++) { for (int y = 0; y < 1 << zoom; y++) { - if (!zoomResult.test(x, y)) + if (!zoomResult->test(x, y)) continue; if (hasClippingBox) { diff --git a/test/tile_coordinates_set.test.cpp b/test/tile_coordinates_set.test.cpp new file mode 100644 index 00000000..87a054b8 --- /dev/null +++ b/test/tile_coordinates_set.test.cpp @@ -0,0 +1,58 @@ +#include +#include "external/minunit.h" +#include "tile_coordinates_set.h" + +MU_TEST(test_tile_coordinates_set) { + { + PreciseTileCoordinatesSet z0(0); + mu_check(z0.test(0, 0) == false); + mu_check(z0.size() == 0); + mu_check(z0.zoom() == 0); + + z0.set(0, 0); + mu_check(z0.test(0, 0) == true); + mu_check(z0.size() == 1); + } + + { + PreciseTileCoordinatesSet z6(6); + mu_check(z6.test(0, 0) == false); + + z6.set(0, 0); + mu_check(z6.test(0, 0) == true); + mu_check(z6.test(1, 0) == false); + mu_check(z6.test(0, 1) == false); + mu_check(z6.size() == 1); + mu_check(z6.zoom() == 6); + } + + // Wrapped sets should extrapolate from lower zooms + { + PreciseTileCoordinatesSet z1(1); + LossyTileCoordinatesSet z2(2, z1); + + mu_check(z2.size() == 0); + for (int x = 0; x < 4; x++) { + for (int y = 0; y < 4; y++) { + mu_check(z2.test(x, y) == false); + } + } + z1.set(0, 0); + mu_check(z2.size() == 4); + mu_check(z2.test(0, 0) == true); + mu_check(z2.test(0, 1) == true); + mu_check(z2.test(1, 0) == true); + mu_check(z2.test(1, 1) == true); + mu_check(z2.test(2, 2) == false); + } +} + +MU_TEST_SUITE(test_suite_tile_coordinates_set) { + MU_RUN_TEST(test_tile_coordinates_set); +} + +int main() { + MU_RUN_SUITE(test_suite_tile_coordinates_set); + MU_REPORT(); + return MU_EXIT_CODE; +}