Remove need_tilestats. Add a bunch of debug logging

serial.hpp

@@ -165,7 +165,6 @@ struct serial_feature {
 	long long clustered;			       // does this feature need the clustered/point_count attributes?
 	const char *stringpool;			       // string pool for keys/values lookup
 	std::shared_ptr<std::string> tile_stringpool;  // string pool for mvt_value construction
-	std::set<std::string> need_tilestats;
 
 	int z;	// tile being produced
 	int tx;

tile.cpp

@@ -1405,10 +1405,6 @@ void add_tilestats(std::string const &layername, int z, std::vector<std::map<std
 }
 
 void promote_attribute(std::string const &key, serial_feature &p, key_pool &key_pool) {
-	if (p.need_tilestats.count(key) == 0) {
-		p.need_tilestats.insert(key);
-	}
-
 	// If the feature being merged into has this key as a metadata reference,
 	// promote it to a full_key so it can be modified
 
@@ -1430,10 +1426,6 @@ void promote_attribute(std::string const &key, serial_feature &p, key_pool &key_
 }
 
 void promote_attribute_prefix(std::string const &key, std::string const &prefixed_key, serial_feature &p, key_pool &key_pool) {
-	if (p.need_tilestats.count(prefixed_key) == 0) {
-		p.need_tilestats.insert(prefixed_key);
-	}
-
 	// does the prefixed attribute already exist as a full key?
 	ssize_t found_as = -1;
 	for (size_t i = 0; i < p.full_keys.size(); i++) {
@@ -1450,6 +1442,7 @@ void promote_attribute_prefix(std::string const &key, std::string const &prefixe
 	if (found_as >= 0) {
 		p.full_keys.push_back(key_pool.pool(prefixed_key));
 		p.full_values.push_back(p.full_values[found_as]);
+
 		return;
 	}
 
@@ -1857,6 +1850,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		strategy strategy;
 		strategy.detail_reduced = detail_reduced;
 
+		fprintf(stderr, "@@@@@ reading features in %d/%d/%d\n", z, tx, ty);
+
 		for (size_t seq = 0;; seq++) {
 			serial_feature sf;
 			ssize_t which_serial_feature = -1;
@@ -2221,6 +2216,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			coalesced_area = 0;
 		}
 
+		fprintf(stderr, "@@@@@ done reading features in %d/%d/%d\n", z, tx, ty);
+
 		// We are done reading the features.
 		// Close the prefilter if it was opened.
 		// Close the output files for the next zoom level.
@@ -2293,15 +2290,17 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 		// Reorder and coalesce.
 		// Sort back into input order or by attribute value
 
+		fprintf(stderr, "@@@@ sorting shared nodes in %d/%d/%d\n", z, tx, ty);
 		std::stable_sort(shared_nodes.begin(), shared_nodes.end());
+		fprintf(stderr, "@@@@ done sorting shared nodes in %d/%d/%d\n", z, tx, ty);
 
 		for (auto &kv : layers) {
 			std::string const &layername = kv.first;
 			std::vector<std::shared_ptr<serial_feature>> &features = kv.second.features;
 
-			if (retain_points_multiplier > 1) {
+			fprintf(stderr, "@@@@ doing retain_points_multiplier in %d/%d/%d\n", z, tx, ty);
-				add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true"));
 
+			if (retain_points_multiplier > 1) {
 				// mapping from input sequence to current sequence within this tile
 				std::vector<std::pair<size_t, size_t>> feature_sequences;
 
@@ -2322,11 +2321,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 
 					features[j]->full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence"));
 					features[j]->full_values.push_back(sv);
-
-					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j]->full_keys.back(), sv);
 				}
 			}
 
+			fprintf(stderr, "@@@@ doing cluster stuff in %d/%d/%d\n", z, tx, ty);
+
 			for (size_t i = 0; i < features.size(); i++) {
 				serial_feature &p = *features[i];
 
@@ -2340,22 +2339,16 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					sv.s = "true";
 					p.full_values.push_back(sv);
 
-					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv);
-
 					p.full_keys.push_back(key_pool.pool("point_count"));
 					sv2.type = mvt_double;
 					sv2.s = std::to_string(point_count);
 					p.full_values.push_back(sv2);
 
-					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2);
-
 					p.full_keys.push_back(key_pool.pool("sqrt_point_count"));
 					sv3.type = mvt_double;
 					sv3.s = std::to_string(round(100 * sqrt(point_count)) / 100.0);
 					p.full_values.push_back(sv3);
 
-					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3);
-
 					p.full_keys.push_back(key_pool.pool("point_count_abbreviated"));
 					sv4.type = mvt_string;
 					if (point_count >= 10000) {
@@ -2367,24 +2360,20 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					}
 					sv4.s = abbrev;
 					p.full_values.push_back(sv4);
-
-					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count_abbreviated", sv4);
 				}
 
-				if (p.need_tilestats.size() > 0) {
+				for (size_t j = 0; j < p.full_keys.size(); j++) {
-					for (size_t j = 0; j < p.full_keys.size(); j++) {
+					// remove accumulation state
-						if (p.need_tilestats.count(*p.full_keys[j]) > 0) {
+					size_t found = p.full_values[j].s.find('\0');
-							// remove accumulation state
+					if (found != std::string::npos) {
-							size_t found = p.full_values[j].s.find('\0');
+						p.full_values[j].s = p.full_values[j].s.substr(0, found);
-							if (found != std::string::npos) {
-								p.full_values[j].s = p.full_values[j].s.substr(0, found);
-							}
-							add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]);
-						}
 					}
+					add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]);
 				}
 			}
 
+			fprintf(stderr, "@@@@ doing detect shared borders in %d/%d/%d\n", z, tx, ty);
+
 			if (additional[A_DETECT_SHARED_BORDERS]) {
 				find_common_edges(features, z, line_detail, simplification, maxzoom, merge_fraction);
 			}
@@ -2394,6 +2383,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				tasks = 1;
 			}
 
+			fprintf(stderr, "@@@@ doing simplification in %d/%d/%d\n", z, tx, ty);
+
 			{
 				pthread_t pthreads[tasks];
 				std::vector<simplification_worker_arg> args;
@@ -2443,10 +2434,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 
 			std::vector<std::shared_ptr<serial_feature>> &layer_features = features;
 
+			fprintf(stderr, "@@@@ doing reorder in %d/%d/%d\n", z, tx, ty);
+
 			if (additional[A_REORDER]) {
 				std::stable_sort(layer_features.begin(), layer_features.end(), coalindexcmp_comparator());
 			}
 
+			fprintf(stderr, "@@@@ doing coalesce in %d/%d/%d\n", z, tx, ty);
+
 			if (additional[A_COALESCE]) {
 				// coalesce adjacent identical features if requested
 
@@ -2471,6 +2466,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				layer_features.resize(out);
 			}
 
+			fprintf(stderr, "@@@@ doing coalesce cleanup in %d/%d/%d\n", z, tx, ty);
+
 			{
 				// clean up coalesced linestrings by simplification
 				// and coalesced polygons by cleaning
@@ -2506,18 +2503,24 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				layer_features.resize(out);
 			}
 
+			fprintf(stderr, "@@@@ doing input order in %d/%d/%d\n", z, tx, ty);
+
 			if (prevent[P_INPUT_ORDER]) {
 				auto clustered = assemble_multiplier_clusters(layer_features);
 				std::stable_sort(clustered.begin(), clustered.end(), preservecmp);
 				layer_features = disassemble_multiplier_clusters(clustered);
 			}
 
+			fprintf(stderr, "@@@@ doing size order in %d/%d/%d\n", z, tx, ty);
+
 			if (order_by.size() != 0) {
 				auto clustered = assemble_multiplier_clusters(layer_features);
 				std::stable_sort(clustered.begin(), clustered.end(), ordercmp());
 				layer_features = disassemble_multiplier_clusters(clustered);
 			}
 
+			fprintf(stderr, "@@@@ doing limit in %d/%d/%d\n", z, tx, ty);
+
 			if (z == maxzoom && limit_tile_feature_count_at_maxzoom != 0) {
 				if (layer_features.size() > limit_tile_feature_count_at_maxzoom) {
 					// this is maxzoom; ok to stop early still because they said to limit abruptly
@@ -2542,6 +2545,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			std::vector<std::shared_ptr<serial_feature>> &layer_features = layer_iterator->second.features;
 			feature_count += layer_features.size();
 
+			fprintf(stderr, "@@@@ doing conversion to mvt_layer %d/%d/%d\n", z, tx, ty);
+
 			mvt_layer layer;
 			layer.name = layer_iterator->first;
 			layer.version = 2;
@@ -2634,6 +2639,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 			continue;
 		}
 
+		fprintf(stderr, "@@@@ doing feature count check mvt_layer %d/%d/%d\n", z, tx, ty);
+
 		// Again, adjust the retabulated feature count to estimate
 		// how many total features there would have been if we hadn't
 		// hit the limit and started dropping early.
@@ -2731,17 +2738,23 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 				}
 			}
 
+			fprintf(stderr, "@@@@ doing tile_encoding %d/%d/%d\n", z, tx, ty);
+
 			std::string compressed;
 			std::string pbf = tile.encode();
 
 			tile.layers.clear();
 
+			fprintf(stderr, "@@@@ doing tile compression %d/%d/%d\n", z, tx, ty);
+
 			if (!prevent[P_TILE_COMPRESSION]) {
 				compress(pbf, compressed, true);
 			} else {
 				compressed = pbf;
 			}
 
+			fprintf(stderr, "@@@@ doing size checks %d/%d/%d\n", z, tx, ty);
+
 			// And similarly, adjust the compressed byte size to estimate
 			// what it would have been if we hadn't stopped dropping features early
 
@@ -2846,12 +2859,16 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
 					exit(EXIT_IMPOSSIBLE);
 				}
 
+				fprintf(stderr, "@@@@ writing to mbtiles %d/%d/%d\n", z, tx, ty);
+
 				if (outdb != NULL) {
 					mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
 				} else if (outdir != NULL) {
 					dir_write_tile(outdir, z, tx, ty, compressed);
 				}
 
+				fprintf(stderr, "@@@@ done writing to mbtiles %d/%d/%d\n", z, tx, ty);
+
 				if (pthread_mutex_unlock(&db_lock) != 0) {
 					perror("pthread_mutex_unlock");
 					exit(EXIT_PTHREAD);