Merge pull request #436 from mapbox/tilestats

Generate layer, feature, and attribute statistics as part of tileset metadata

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.21.0
+
+* Generate layer, feature, and attribute statistics as part of tileset metadata
+
 ## 1.20.1
 
 * Close mbtiles file properly when there are no valid features in the input

Makefile

@@ -55,7 +55,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o
+tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 unit: unit.o text.o
@@ -85,7 +85,7 @@ test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test p
 %.json.check:
 	./tippecanoe -aD -f -o $@.mbtiles $(subst @,:,$(subst %,/,$(subst _, ,$(patsubst %.json.check,%,$(word 4,$(subst /, ,$@)))))) $(wildcard $(subst $(SPACE),/,$(wordlist 1,2,$(subst /, ,$@)))/*.json) < /dev/null
 	./tippecanoe-decode $@.mbtiles > $@.out
-	cmp $(patsubst %.check,%,$@) $@.out
+	cmp $@.out $(patsubst %.check,%,$@)
 	rm $@.out $@.mbtiles
 
 parallel-test:
@@ -115,7 +115,7 @@ parallel-test:
 	rm tests/parallel/*.mbtiles tests/parallel/*.json
 
 raw-tiles-test:	
-	./tippecanoe -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC
+	./tippecanoe -f -e tests/raw-tiles/raw-tiles tests/raw-tiles/hackspots.geojson -pC
 	diff -x '*.DS_Store' -rq tests/raw-tiles/raw-tiles tests/raw-tiles/compare
 	rm -rf tests/raw-tiles/raw-tiles
 
@@ -163,25 +163,25 @@ join-test:
 	./tippecanoe-decode tests/join-population/no-macarthur.mbtiles > tests/join-population/no-macarthur.mbtiles.json.check
 	cmp tests/join-population/just-macarthur.mbtiles.json.check tests/join-population/just-macarthur.mbtiles.json
 	cmp tests/join-population/no-macarthur.mbtiles.json.check tests/join-population/no-macarthur.mbtiles.json	
-	./tile-join --no-tile-compression -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
+	./tile-join --no-tile-compression -f -e tests/join-population/raw-merged-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
 	diff -x '*.DS_Store' -rq tests/join-population/raw-merged-folder tests/join-population/raw-merged-folder-compare
-	./tippecanoe -z12 -e tests/join-population/tabblock_06001420-folder tests/join-population/tabblock_06001420.json
-	./tippecanoe -Z5 -z10 -e tests/join-population/macarthur-folder -l macarthur tests/join-population/macarthur.json
-	./tippecanoe -d10 -D10 -Z9 -z11 -e tests/join-population/macarthur2-folder -l macarthur tests/join-population/macarthur2.json
+	./tippecanoe -z12 -f -e tests/join-population/tabblock_06001420-folder tests/join-population/tabblock_06001420.json
+	./tippecanoe -Z5 -z10 -f -e tests/join-population/macarthur-folder -l macarthur tests/join-population/macarthur.json
+	./tippecanoe -d10 -D10 -Z9 -z11 -f -e tests/join-population/macarthur2-folder -l macarthur tests/join-population/macarthur2.json
 	./tile-join -f -o tests/join-population/merged-folder.mbtiles tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder
 	./tippecanoe-decode tests/join-population/merged-folder.mbtiles > tests/join-population/merged-folder.mbtiles.json.check
 	cmp tests/join-population/merged-folder.mbtiles.json.check tests/join-population/merged-folder.mbtiles.json
-	./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-mbtiles-to-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
-	./tile-join -n "merged name" -N "merged description" -e tests/join-population/merged-folders-to-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder
+	./tile-join -n "merged name" -N "merged description" -f -e tests/join-population/merged-mbtiles-to-folder tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
+	./tile-join -n "merged name" -N "merged description" -f -e tests/join-population/merged-folders-to-folder tests/join-population/tabblock_06001420-folder tests/join-population/macarthur-folder tests/join-population/macarthur2-folder
 	diff -x '*.DS_Store' -rq tests/join-population/merged-mbtiles-to-folder tests/join-population/merged-folders-to-folder
 	./tile-join -f -c tests/join-population/windows.csv -o tests/join-population/windows-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
-	./tile-join -c tests/join-population/windows.csv -e tests/join-population/windows-merged-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -c tests/join-population/windows.csv -f -e tests/join-population/windows-merged-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
 	./tile-join -f -o tests/join-population/windows-merged2.mbtiles tests/join-population/windows-merged-folder
 	./tippecanoe-decode tests/join-population/windows-merged.mbtiles > tests/join-population/windows-merged.mbtiles.json.check
 	./tippecanoe-decode tests/join-population/windows-merged2.mbtiles > tests/join-population/windows-merged2.mbtiles.json.check
 	cmp tests/join-population/windows-merged.mbtiles.json.check tests/join-population/windows-merged2.mbtiles.json.check
 	./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
-	./tile-join -e tests/join-population/macarthur-and-macarthur2-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
+	./tile-join -f -e tests/join-population/macarthur-and-macarthur2-folder tests/join-population/macarthur.mbtiles tests/join-population/macarthur2-folder
 	./tile-join -f -o tests/join-population/macarthur-and-macarthur2-merged2.mbtiles tests/join-population/macarthur-and-macarthur2-folder
 	./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged.mbtiles > tests/join-population/macarthur-and-macarthur2-merged.mbtiles.json.check
 	./tippecanoe-decode tests/join-population/macarthur-and-macarthur2-merged2.mbtiles > tests/join-population/macarthur-and-macarthur2-merged2.mbtiles.json.check

dirtiles.cpp

@@ -2,6 +2,10 @@
 #include <fstream>
 #include <sstream>
 #include <string>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
 #include <sys/stat.h>
 #include "dirtiles.hpp"
 
@@ -24,7 +28,99 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const
 	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
 	newdir = newdir + "/" + std::to_string(ty) + ".pbf";
 
+	struct stat st;
+	if (stat(newdir.c_str(), &st) == 0) {
+		fprintf(stderr, "Can't write tile to already existing %s\n", newdir.c_str());
+		exit(EXIT_FAILURE);
+	}
+
 	std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary);
 	pbfFile.write(pbf.data(), pbf.size());
 	pbfFile.close();
 }
+
+bool numeric(const char *s) {
+	if (*s == '\0') {
+		return false;
+	}
+	for (; *s != 0; s++) {
+		if (*s < '0' || *s > '9') {
+			return false;
+		}
+	}
+	return true;
+}
+
+bool pbfname(const char *s) {
+	while (*s >= '0' && *s <= '9') {
+		s++;
+	}
+
+	return strcmp(s, ".pbf") == 0;
+}
+
+void check_dir(const char *dir, bool rm) {
+	struct stat st;
+
+	std::string meta = std::string(dir) + "/" + "metadata.json";
+	if (rm) {
+		unlink(meta.c_str());  // error OK since it may not exist;
+	} else {
+		if (stat(meta.c_str(), &st) == 0) {
+			fprintf(stderr, "%s: file exists\n", meta.c_str());
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	DIR *d1 = opendir(dir);
+	if (d1 != NULL) {
+		struct dirent *dp;
+		while ((dp = readdir(d1)) != NULL) {
+			if (numeric(dp->d_name)) {
+				std::string z = std::string(dir) + "/" + dp->d_name;
+
+				DIR *d2 = opendir(z.c_str());
+				if (d2 == NULL) {
+					perror(z.c_str());
+					exit(EXIT_FAILURE);
+				}
+
+				struct dirent *dp2;
+				while ((dp2 = readdir(d2)) != NULL) {
+					if (numeric(dp2->d_name)) {
+						std::string x = z + "/" + dp2->d_name;
+
+						DIR *d3 = opendir(x.c_str());
+						if (d3 == NULL) {
+							perror(x.c_str());
+							exit(EXIT_FAILURE);
+						}
+
+						struct dirent *dp3;
+						while ((dp3 = readdir(d3)) != NULL) {
+							if (pbfname(dp3->d_name)) {
+								std::string y = x + "/" + dp3->d_name;
+
+								if (rm) {
+									if (unlink(y.c_str()) != 0) {
+										perror(y.c_str());
+										exit(EXIT_FAILURE);
+									}
+								} else {
+									fprintf(stderr, "%s: file exists\n", y.c_str());
+									exit(EXIT_FAILURE);
+								}
+							}
+						}
+
+						closedir(d3);
+					}
+				}
+
+				closedir(d2);
+			}
+		}
+
+		closedir(d1);
+	}
+}

dirtiles.hpp

@@ -1,3 +1,12 @@
+#include <string>
+
+#ifndef DIRTILES_HPP
+#define DIRTILES_HPP
+
 std::string dir_read_tile(std::string pbfPath);
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
+
+void check_dir(const char *d, bool rm);
+
+#endif

geojson.cpp

@@ -24,11 +24,7 @@
 #include <set>
 #include <map>
 #include <string>
-
-extern "C" {
 #include "jsonpull/jsonpull.h"
-}
-
 #include "pool.hpp"
 #include "projection.hpp"
 #include "memfile.hpp"
@@ -299,10 +295,29 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 		if (ai != layermap->end()) {
 			layer = ai->second.id;
 			layername = tippecanoe_layername;
+
+			if (mb_geometry[t] == VT_POINT) {
+				ai->second.points++;
+			} else if (mb_geometry[t] == VT_LINE) {
+				ai->second.lines++;
+			} else if (mb_geometry[t] == VT_POLYGON) {
+				ai->second.polygons++;
+			}
 		} else {
 			fprintf(stderr, "Internal error: can't find layer name %s\n", tippecanoe_layername.c_str());
 			exit(EXIT_FAILURE);
 		}
+	} else {
+		auto fk = layermap->find(layername);
+		if (fk != layermap->end()) {
+			if (mb_geometry[t] == VT_POINT) {
+				fk->second.points++;
+			} else if (mb_geometry[t] == VT_LINE) {
+				fk->second.lines++;
+			} else if (mb_geometry[t] == VT_POLYGON) {
+				fk->second.polygons++;
+			}
+		}
 	}
 
 	size_t nprop = 0;
@@ -358,8 +373,12 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 						vt = JSON_STRING;
 					} else if (a->second == mvt_float) {
 						vt = JSON_NUMBER;
+						val = std::to_string(atof(val.c_str()));
 					} else if (a->second == mvt_int) {
 						vt = JSON_NUMBER;
+						if (val.size() == 0) {
+							val = "0";
+						}
 
 						for (size_t ii = 0; ii < val.size(); ii++) {
 							char c = val[ii];
@@ -410,8 +429,12 @@ int serialize_geometry(json_object *geometry, json_object *properties, json_obje
 			}
 
 			if (tas.type >= 0) {
+				type_and_string attrib;
+				attrib.type = metatype[m - 1];
+				attrib.string = metaval[m - 1];
+
 				auto fk = layermap->find(layername);
-				fk->second.file_keys.insert(tas);
+				add_to_file_keys(fk->second.file_keys, metakey[m - 1], attrib);
 			}
 		}
 	}

geojson.hpp

@@ -1,3 +1,13 @@
+#ifndef GEOJSON_HPP
+#define GEOJSON_HPP
+
+#include <stdio.h>
+#include <set>
+#include <map>
+#include <string>
+#include "mbtiles.hpp"
+#include "jsonpull/jsonpull.h"
+
 struct parse_json_args {
 	json_pull *jp;
 	const char *reading;
@@ -39,3 +49,5 @@ void json_end_map(struct json_pull *jp);
 
 void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, int maxzoom, std::map<std::string, layermap_entry> *layermap, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types, double *dist_sum, size_t *dist_count, bool want_dist);
 void *run_parse_json(void *v);
+
+#endif

geometry.hpp

@@ -1,3 +1,9 @@
+#ifndef GEOMETRY_HPP
+#define GEOMETRY_HPP
+
+#include <vector>
+#include <sqlite3.h>
+
 #define VT_POINT 1
 #define VT_LINE 2
 #define VT_POLYGON 3
@@ -68,3 +74,5 @@ std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms);
 void check_polygon(drawvec &geom, drawvec &before);
 double get_area(drawvec &geom, size_t i, size_t j);
 double get_mp_area(drawvec &geom);
+
+#endif

jsonpull/jsonpull.h

@@ -1,3 +1,10 @@
+#ifndef JSONPULL_H
+#define JSONPULL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef enum json_type {
 	// These types can be returned by json_read()
 	JSON_HASH,
@@ -65,3 +72,9 @@ void json_disconnect(json_object *j);
 json_object *json_hash_get(json_object *o, const char *s);
 
 char *json_stringify(json_object *o);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif