From 887e2bb1e610b42603abc710386462a1717abaed Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Fri, 31 Jan 2025 10:24:54 -0800 Subject: [PATCH] Remove `need_tilestats`. Add a bunch of debug logging --- serial.hpp | 1 - tile.cpp | 77 +++++++++++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/serial.hpp b/serial.hpp index 1c86cc05..a351f21d 100644 --- a/serial.hpp +++ b/serial.hpp @@ -165,7 +165,6 @@ struct serial_feature { long long clustered; // does this feature need the clustered/point_count attributes? const char *stringpool; // string pool for keys/values lookup std::shared_ptr tile_stringpool; // string pool for mvt_value construction - std::set need_tilestats; int z; // tile being produced int tx; diff --git a/tile.cpp b/tile.cpp index 6d23cbee..82088aaf 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1405,10 +1405,6 @@ void add_tilestats(std::string const &layername, int z, std::vector= 0) { p.full_keys.push_back(key_pool.pool(prefixed_key)); p.full_values.push_back(p.full_values[found_as]); + return; } @@ -1857,6 +1850,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch strategy strategy; strategy.detail_reduced = detail_reduced; + fprintf(stderr, "@@@@@ reading features in %d/%d/%d\n", z, tx, ty); + for (size_t seq = 0;; seq++) { serial_feature sf; ssize_t which_serial_feature = -1; @@ -2221,6 +2216,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch coalesced_area = 0; } + fprintf(stderr, "@@@@@ done reading features in %d/%d/%d\n", z, tx, ty); + // We are done reading the features. // Close the prefilter if it was opened. // Close the output files for the next zoom level. @@ -2293,15 +2290,17 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch // Reorder and coalesce. // Sort back into input order or by attribute value + fprintf(stderr, "@@@@ sorting shared nodes in %d/%d/%d\n", z, tx, ty); std::stable_sort(shared_nodes.begin(), shared_nodes.end()); + fprintf(stderr, "@@@@ done sorting shared nodes in %d/%d/%d\n", z, tx, ty); for (auto &kv : layers) { std::string const &layername = kv.first; std::vector> &features = kv.second.features; - if (retain_points_multiplier > 1) { - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe:retain_points_multiplier_first", serial_val(mvt_bool, "true")); + fprintf(stderr, "@@@@ doing retain_points_multiplier in %d/%d/%d\n", z, tx, ty); + if (retain_points_multiplier > 1) { // mapping from input sequence to current sequence within this tile std::vector> feature_sequences; @@ -2322,11 +2321,11 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch features[j]->full_keys.push_back(key_pool.pool("tippecanoe:retain_points_multiplier_sequence")); features[j]->full_values.push_back(sv); - - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *features[j]->full_keys.back(), sv); } } + fprintf(stderr, "@@@@ doing cluster stuff in %d/%d/%d\n", z, tx, ty); + for (size_t i = 0; i < features.size(); i++) { serial_feature &p = *features[i]; @@ -2340,22 +2339,16 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch sv.s = "true"; p.full_values.push_back(sv); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv); - p.full_keys.push_back(key_pool.pool("point_count")); sv2.type = mvt_double; sv2.s = std::to_string(point_count); p.full_values.push_back(sv2); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2); - p.full_keys.push_back(key_pool.pool("sqrt_point_count")); sv3.type = mvt_double; sv3.s = std::to_string(round(100 * sqrt(point_count)) / 100.0); p.full_values.push_back(sv3); - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3); - p.full_keys.push_back(key_pool.pool("point_count_abbreviated")); sv4.type = mvt_string; if (point_count >= 10000) { @@ -2367,24 +2360,20 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } sv4.s = abbrev; p.full_values.push_back(sv4); - - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count_abbreviated", sv4); } - if (p.need_tilestats.size() > 0) { - for (size_t j = 0; j < p.full_keys.size(); j++) { - if (p.need_tilestats.count(*p.full_keys[j]) > 0) { - // remove accumulation state - size_t found = p.full_values[j].s.find('\0'); - if (found != std::string::npos) { - p.full_values[j].s = p.full_values[j].s.substr(0, found); - } - add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); - } + for (size_t j = 0; j < p.full_keys.size(); j++) { + // remove accumulation state + size_t found = p.full_values[j].s.find('\0'); + if (found != std::string::npos) { + p.full_values[j].s = p.full_values[j].s.substr(0, found); } + add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); } } + fprintf(stderr, "@@@@ doing detect shared borders in %d/%d/%d\n", z, tx, ty); + if (additional[A_DETECT_SHARED_BORDERS]) { find_common_edges(features, z, line_detail, simplification, maxzoom, merge_fraction); } @@ -2394,6 +2383,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch tasks = 1; } + fprintf(stderr, "@@@@ doing simplification in %d/%d/%d\n", z, tx, ty); + { pthread_t pthreads[tasks]; std::vector args; @@ -2443,10 +2434,14 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch std::vector> &layer_features = features; + fprintf(stderr, "@@@@ doing reorder in %d/%d/%d\n", z, tx, ty); + if (additional[A_REORDER]) { std::stable_sort(layer_features.begin(), layer_features.end(), coalindexcmp_comparator()); } + fprintf(stderr, "@@@@ doing coalesce in %d/%d/%d\n", z, tx, ty); + if (additional[A_COALESCE]) { // coalesce adjacent identical features if requested @@ -2471,6 +2466,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch layer_features.resize(out); } + fprintf(stderr, "@@@@ doing coalesce cleanup in %d/%d/%d\n", z, tx, ty); + { // clean up coalesced linestrings by simplification // and coalesced polygons by cleaning @@ -2506,18 +2503,24 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch layer_features.resize(out); } + fprintf(stderr, "@@@@ doing input order in %d/%d/%d\n", z, tx, ty); + if (prevent[P_INPUT_ORDER]) { auto clustered = assemble_multiplier_clusters(layer_features); std::stable_sort(clustered.begin(), clustered.end(), preservecmp); layer_features = disassemble_multiplier_clusters(clustered); } + fprintf(stderr, "@@@@ doing size order in %d/%d/%d\n", z, tx, ty); + if (order_by.size() != 0) { auto clustered = assemble_multiplier_clusters(layer_features); std::stable_sort(clustered.begin(), clustered.end(), ordercmp()); layer_features = disassemble_multiplier_clusters(clustered); } + fprintf(stderr, "@@@@ doing limit in %d/%d/%d\n", z, tx, ty); + if (z == maxzoom && limit_tile_feature_count_at_maxzoom != 0) { if (layer_features.size() > limit_tile_feature_count_at_maxzoom) { // this is maxzoom; ok to stop early still because they said to limit abruptly @@ -2542,6 +2545,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch std::vector> &layer_features = layer_iterator->second.features; feature_count += layer_features.size(); + fprintf(stderr, "@@@@ doing conversion to mvt_layer %d/%d/%d\n", z, tx, ty); + mvt_layer layer; layer.name = layer_iterator->first; layer.version = 2; @@ -2634,6 +2639,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch continue; } + fprintf(stderr, "@@@@ doing feature count check mvt_layer %d/%d/%d\n", z, tx, ty); + // Again, adjust the retabulated feature count to estimate // how many total features there would have been if we hadn't // hit the limit and started dropping early. @@ -2731,17 +2738,23 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } + fprintf(stderr, "@@@@ doing tile_encoding %d/%d/%d\n", z, tx, ty); + std::string compressed; std::string pbf = tile.encode(); tile.layers.clear(); + fprintf(stderr, "@@@@ doing tile compression %d/%d/%d\n", z, tx, ty); + if (!prevent[P_TILE_COMPRESSION]) { compress(pbf, compressed, true); } else { compressed = pbf; } + fprintf(stderr, "@@@@ doing size checks %d/%d/%d\n", z, tx, ty); + // And similarly, adjust the compressed byte size to estimate // what it would have been if we hadn't stopped dropping features early @@ -2846,12 +2859,16 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch exit(EXIT_IMPOSSIBLE); } + fprintf(stderr, "@@@@ writing to mbtiles %d/%d/%d\n", z, tx, ty); + if (outdb != NULL) { mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size()); } else if (outdir != NULL) { dir_write_tile(outdir, z, tx, ty, compressed); } + fprintf(stderr, "@@@@ done writing to mbtiles %d/%d/%d\n", z, tx, ty); + if (pthread_mutex_unlock(&db_lock) != 0) { perror("pthread_mutex_unlock"); exit(EXIT_PTHREAD);