mod_tile: Update to a new meta tile scheme which stores all sub-tiles…

… in a single .meta file. The PNG files are extracted from this on the fly by mod_tile. This is more efficient in disk space and inode usage

Makefile

@@ -17,11 +17,11 @@ EXTRA_CFLAGS = -I$(builddir)
 EXTRA_CPPFLAGS += -g -O2 -Wall
 EXTRA_LDFLAGS += $(shell pkg-config --libs libagg)
 
-all: local-shared-build renderd speedtest render_list render_old
+all: local-shared-build renderd speedtest render_list render_old convert_meta
 
 clean:
 	rm -f *.o *.lo *.slo *.la .libs/*
-	rm -f renderd render_list speedtest render_old
+	rm -f renderd render_list speedtest render_old convert_meta
 
 RENDER_CPPFLAGS += -g -O2 -Wall
 RENDER_CPPFLAGS += -I/usr/local/include/mapnik
@@ -35,7 +35,7 @@ RENDER_LDFLAGS += $(shell pkg-config --libs freetype2)
 #RENDER_LDFLAGS += $(shell Magick++-config --ldflags --libs)
 RENDER_LDFLAGS += $(shell pkg-config --libs libagg)
 
-renderd: daemon.c gen_tile.cpp dir_utils.c protocol.h render_config.h dir_utils.h
+renderd: store.c daemon.c gen_tile.cpp dir_utils.c protocol.h render_config.h dir_utils.h store.h
 	$(CXX) -o $@ $^ $(RENDER_LDFLAGS) $(RENDER_CPPFLAGS)
 
 speedtest: render_config.h protocol.h dir_utils.c dir_utils.h
@@ -44,6 +44,8 @@ render_list: render_config.h protocol.h dir_utils.c dir_utils.h
 
 render_old: render_config.h protocol.h dir_utils.c dir_utils.h
 
+convert_meta: render_config.h protocol.h dir_utils.c dir_utils.h store.c
+
 MYSQL_CFLAGS += -g -O2 -Wall
 MYSQL_CFLAGS += $(shell mysql_config --cflags)
 

convert_meta.c

@@ -0,0 +1,164 @@
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/un.h>
+#include <poll.h>
+#include <errno.h>
+#include <math.h>
+#include <getopt.h>
+#include <time.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <assert.h>
+#include <fcntl.h>
+
+#include "render_config.h"
+#include "dir_utils.h"
+#include "store.h"
+
+static int minZoom = 0;
+static int maxZoom = 18;
+static int verbose = 0;
+static int num_render = 0, num_all = 0;
+static struct timeval start, end;
+static int unpack;
+
+void display_rate(struct timeval start, struct timeval end, int num) 
+{
+    int d_s, d_us;
+    float sec;
+
+    d_s  = end.tv_sec  - start.tv_sec;
+    d_us = end.tv_usec - start.tv_usec;
+
+    sec = d_s + d_us / 1000000.0;
+
+    printf("Converted %d tiles in %.2f seconds (%.2f tiles/s)\n", num, sec, num / sec);
+    fflush(NULL);
+}
+
+static void descend(const char *search)
+{
+    DIR *tiles = opendir(search);
+    struct dirent *entry;
+    char path[PATH_MAX];
+
+    if (!tiles) {
+        //fprintf(stderr, "Unable to open directory: %s\n", search);
+        return;
+    }
+
+    while ((entry = readdir(tiles))) {
+        struct stat b;
+        char *p;
+
+        //check_load();
+
+        if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+            continue;
+        snprintf(path, sizeof(path), "%s/%s", search, entry->d_name);
+        if (stat(path, &b))
+            continue;
+        if (S_ISDIR(b.st_mode)) {
+            descend(path);
+            continue;
+        }
+        p = strrchr(path, '.');
+        if (p) {
+            if (unpack) {
+                if (!strcmp(p, ".meta")) 
+                    process_unpack(path);
+            } else {
+                if (!strcmp(p, ".png")) 
+                  process_pack(path);
+            }
+        }
+    }
+    closedir(tiles);
+}
+
+
+int main(int argc, char **argv)
+{
+    int z, c;
+
+    while (1) {
+        int option_index = 0;
+        static struct option long_options[] = {
+            {"min-zoom", 1, 0, 'z'},
+            {"max-zoom", 1, 0, 'Z'},
+            {"unpack", 0, 0, 'u'},
+            {"verbose", 0, 0, 'v'},
+            {"help", 0, 0, 'h'},
+            {0, 0, 0, 0}
+        };
+
+        c = getopt_long(argc, argv, "uhvz:Z:", long_options, &option_index);
+        if (c == -1)
+            break;
+
+        switch (c) {
+            case 'z':
+                minZoom=atoi(optarg);
+                if (minZoom < 0 || minZoom > 18) {
+                    fprintf(stderr, "Invalid minimum zoom selected, must be between 0 and 18\n");
+                    return 1;
+                }
+                break;
+            case 'Z':
+                maxZoom=atoi(optarg);
+                if (maxZoom < 0 || maxZoom > 18) {
+                    fprintf(stderr, "Invalid maximum zoom selected, must be between 0 and 18\n");
+                    return 1;
+                }
+                break;
+            case 'u':
+                unpack=1;
+                break;
+            case 'v':
+                verbose=1;
+                break;
+            case 'h':
+                fprintf(stderr, "Convert the rendered PNGs into the more efficient .meta format\n");
+                fprintf(stderr, "\t-u|--unpack\tUnpack the .meta files back to PNGs\n");
+                fprintf(stderr, "\t-z|--min-zoom\tonly process tiles greater or equal this zoom level (default 0)\n");
+                fprintf(stderr, "\t-Z|--max-zoom\tonly process tiles less than or equal to this zoom level (default 18)\n");
+                return -1;
+            default:
+                fprintf(stderr, "unhandled char '%c'\n", c);
+                break;
+        }
+    }
+
+    if (maxZoom < minZoom) {
+        fprintf(stderr, "Invalid zoom range, max zoom must be greater or equal to minimum zoom\n");
+        return 1;
+    }
+
+    fprintf(stderr, "Converting tiles\n");
+
+    gettimeofday(&start, NULL);
+
+    for (z=minZoom; z<=maxZoom; z++) {
+        char path[PATH_MAX];
+        snprintf(path, PATH_MAX, WWW_ROOT HASH_PATH "/%d", z);
+        descend(path);
+    }
+
+    gettimeofday(&end, NULL);
+    printf("\nTotal for all tiles converted\n");
+    printf("Meta tiles converted: ");
+    display_rate(start, end, num_render);
+    printf("Total tiles converted: ");
+    display_rate(start, end, num_render * METATILE * METATILE);
+    printf("Total tiles handled: ");
+    display_rate(start, end, num_all);
+
+    return 0;
+}

daemon.c

@@ -176,18 +176,15 @@ enum protoCmd rx_request(const struct protocol *req, int fd)
         return cmdIgnore;
     }
 
-    fprintf(stderr, "%s fd(%d) z(%d), x(%d), y(%d), path(%s)\n",
-            cmdStr(req->cmd), fd, req->z, req->x, req->y, req->path);
+    fprintf(stderr, "%s fd(%d) z(%d), x(%d), y(%d)\n",
+            cmdStr(req->cmd), fd, req->z, req->x, req->y);
 
     if ((req->cmd != cmdRender) && (req->cmd != cmdDirty))
         return cmdIgnore;
 
     if (check_xyz(req->x, req->y, req->z))
         return cmdNotDone;
 
-    if (mkdirp(req->path))
-        return cmdNotDone;
-
     item = (struct item *)malloc(sizeof(*item));
     if (!item) {
             fprintf(stderr, "malloc failed\n");

dir_utils.c

@@ -101,7 +101,7 @@ int path_to_xyz(const char *path, int *px, int *py, int *pz)
 {
     int i, n, hash[5], x, y, z;
 
-    n = sscanf(path, WWW_ROOT HASH_PATH "/%d/%d/%d/%d/%d/%d.png", pz, &hash[0], &hash[1], &hash[2], &hash[3], &hash[4]);
+    n = sscanf(path, WWW_ROOT HASH_PATH "/%d/%d/%d/%d/%d/%d", pz, &hash[0], &hash[1], &hash[2], &hash[3], &hash[4]);
     if (n != 6) {
         fprintf(stderr, "Failed to parse tile path: %s\n", path);
         return 1;
@@ -123,3 +123,24 @@ int path_to_xyz(const char *path, int *px, int *py, int *pz)
         return check_xyz(x, y, z);
     }
 }
+
+// Returns the path to the meta-tile and the offset within the meta-tile
+int xyz_to_meta(char *path, size_t len, int x, int y, int z)
+{
+    unsigned char i, hash[5], offset, mask;
+
+    // Each meta tile winds up in its own file, with several in each leaf directory
+    // the .meta tile name is beasd on the sub-tile at (0,0)
+    mask = METATILE - 1;
+    offset = (x & mask) * METATILE + (y & mask);
+    x &= ~mask;
+    y &= ~mask;
+
+    for (i=0; i<5; i++) {
+        hash[i] = ((x & 0x0f) << 4) | (y & 0x0f);
+        x >>= 4;
+        y >>= 4;
+    }
+    snprintf(path, len, WWW_ROOT HASH_PATH "/%d/%u/%u/%u/%u/%u.meta", z, hash[4], hash[3], hash[2], hash[1], hash[0]);
+    return offset;
+}

dir_utils.h

@@ -23,6 +23,12 @@ const char *xyz_to_path(char *path, size_t len, int x, int y, int z);
 int check_xyz(int x, int y, int z);
 int path_to_xyz(const char *path, int *px, int *py, int *pz);
 
+/* New meta-tile storage functions */
+
+/* Returns the path to the meta-tile and the offset within the meta-tile */
+int xyz_to_meta(char *path, size_t len, int x, int y, int z);
+
+
 #ifdef __cplusplus
   }
 #endif

gen_tile.cpp

@@ -18,7 +18,7 @@
 #include "protocol.h"
 #include "render_config.h"
 #include "dir_utils.h"
-
+#include "store.h"
 
 using namespace mapnik;
 
@@ -241,6 +241,7 @@ void *render_thread(__attribute__((unused)) void *unused)
             unsigned int size = MIN(METATILE, 1 << req->z);
     //pthread_mutex_lock(&map_lock);
             ret = render(m, item->mx, item->my, req->z, size);
+            process_meta(item->mx, item->my, req->z);
     //pthread_mutex_unlock(&map_lock);
 
 #else