Compare commits

...
7 Commits
10 changed files with 157 additions and 172 deletions
+3 -2
View File
@@ -1,14 +1,15 @@
PREFIX ?= /usr/local
MANDIR ?= $(PREFIX)/share/man/man1/
BUILDTYPE ?= Release
BUILD_INFO ?=
SHELL = /bin/sh
# inherit from env if set
CC := $(CC)
CXX := $(CXX)
CFLAGS := $(CFLAGS) -fPIE
CXXFLAGS := $(CXXFLAGS) -std=c++17 -fPIE
CFLAGS := $(CFLAGS) -fPIE -DBUILD_INFO=$(BUILD_INFO)
CXXFLAGS := $(CXXFLAGS) -std=c++17 -fPIE -DBUILD_INFO=$(BUILD_INFO)
LDFLAGS := $(LDFLAGS)
WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
RELEASE_FLAGS := -O3 -DNDEBUG
+37 -31
View File
@@ -1188,37 +1188,6 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
static const std::string retain_points_multiplier_sequence = "tippecanoe:retain_points_multiplier_sequence";
for (auto feature : layer.features) {
bool flush_multiplier_cluster = false;
if (demultiply) {
for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) {
mvt_value v = layer.values[feature.tags[i + 1]];
if (v.type == mvt_bool && v.numeric_value.bool_value) {
flush_multiplier_cluster = true;
feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
}
} else if (i < (ssize_t) feature.tags.size() && layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) {
mvt_value v = layer.values[feature.tags[i + 1]];
feature.seq = mvt_value_to_long_long(v);
feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
}
}
} else {
flush_multiplier_cluster = true;
}
if (flush_multiplier_cluster) {
if (pending_tile_features.size() > 0) {
feature_out(pending_tile_features, *outlayer, keep, attribute_accum, tile_stringpool);
pending_tile_features.clear();
}
}
std::set<std::string> exclude_attributes;
if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) {
continue;
}
drawvec geom;
int t = feature.type;
@@ -1268,6 +1237,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
long long b = outtilesize * buffer / 256;
if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
// quick exclusion by bounding box
continue;
}
@@ -1281,6 +1251,42 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
}
}
if (geom.size() == 0) {
// clipped away
continue;
}
bool flush_multiplier_cluster = false;
if (demultiply) {
for (ssize_t i = feature.tags.size() - 2; i >= 0; i -= 2) {
if (layer.keys[feature.tags[i]] == retain_points_multiplier_first) {
mvt_value v = layer.values[feature.tags[i + 1]];
if (v.type == mvt_bool && v.numeric_value.bool_value) {
flush_multiplier_cluster = true;
feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
}
} else if (i < (ssize_t) feature.tags.size() && layer.keys[feature.tags[i]] == retain_points_multiplier_sequence) {
mvt_value v = layer.values[feature.tags[i + 1]];
feature.seq = mvt_value_to_long_long(v);
feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
}
}
} else {
flush_multiplier_cluster = true;
}
if (flush_multiplier_cluster) {
if (pending_tile_features.size() > 0) {
feature_out(pending_tile_features, *outlayer, keep, attribute_accum, tile_stringpool);
pending_tile_features.clear();
}
}
std::set<std::string> exclude_attributes;
if (filter != NULL && !evaluate(feature, layer, filter, exclude_attributes, nz, unidecode_data)) {
continue;
}
bool still_need_simplification_after_reduction = false;
if (t == VT_POLYGON && tiny_polygon_size > 0) {
bool simplified_away_by_reduction = false;
+1 -2
View File
@@ -49,7 +49,6 @@
#include "tile.hpp"
#include "pool.hpp"
#include "projection.hpp"
#include "version.hpp"
#include "memfile.hpp"
#include "main.hpp"
#include "geojson.hpp"
@@ -3608,7 +3607,7 @@ int main(int argc, char **argv) {
}
case 'v':
fprintf(stderr, "tippecanoe %s\n", VERSION);
fprintf(stderr, "tippecanoe %s\n", version_str().c_str());
exit(EXIT_SUCCESS);
case 'P':
+12 -1
View File
@@ -654,6 +654,17 @@ static double sixdig(double val) {
return std::round(val * 1e6) / 1e6;
}
#define str(x) #x
#define xstr(x) str(x)
std::string version_str() {
std::string s = VERSION;
std::string build_info = xstr(BUILD_INFO);
if (build_info.size() > 0) {
s += " " + build_info;
}
return s;
}
metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies, int basezoom, double droprate, int retain_points_multiplier) {
metadata m;
@@ -684,7 +695,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
m.attribution = attribution;
}
m.generator = program + " " + VERSION;
m.generator = program + " " + version_str();
m.generator_options = commandline;
m.strategies_json = stringify_strategies(strategies);
+1
View File
@@ -77,5 +77,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &layername, serial_val const &val);
std::string version_str();
#endif
File diff suppressed because one or more lines are too long
+4 -3
View File
@@ -1,14 +1,15 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "0.000000,85.051129,0.000000,85.051129",
"bounds": "-180.000000,85.051129,180.000000,85.051129",
"center": "-179.989014,85.051129,14",
"center": "0.000000,85.051129,0",
"description": "tests/raw-tiles/nothing",
"format": "pbf",
"generator_options": "./tippecanoe -q -f -e tests/raw-tiles/nothing tests/raw-tiles/nothing.geojson",
"json": "{\"vector_layers\":[{\"id\":\"nothing\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":14,\"fields\":{}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"nothing\",\"count\":1,\"geometry\":\"Point\",\"attributeCount\":0,\"attributes\":[]}]}}",
"maxzoom": "14",
"json": "{\"vector_layers\":[{\"id\":\"nothing\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"nothing\",\"count\":1,\"geometry\":\"Point\",\"attributeCount\":0,\"attributes\":[]}]}}",
"maxzoom": "0",
"minzoom": "0",
"name": "tests/raw-tiles/nothing",
"tippecanoe_decisions": "{\"basezoom\":14,\"droprate\":2.5,\"retain_points_multiplier\":1}",
"type": "overlay",
"version": "2"
}, "features": [
+57 -24
View File
@@ -1521,7 +1521,7 @@ struct layer_features {
size_t multiplier_cluster_size = 0; // The feature count of the current multiplier cluster
};
bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector<std::vector<std::string>> *layer_unmaps, size_t &multiplier_seq, strategy &strategy, bool &drop_rest, std::unordered_map<std::string, attribute_op> const *attribute_accum) {
bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features &layer, serial_feature &sf, std::vector<std::vector<std::string>> *layer_unmaps, size_t &multiplier_seq, strategy &strategy, bool &drop_rest_of_cluster, std::unordered_map<std::string, attribute_op> const *attribute_accum) {
ssize_t which_serial_feature;
if (find_feature_to_accumulate_onto(layer.features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
@@ -1533,7 +1533,7 @@ bool drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer_features
} else {
preserve_attributes(attribute_accum, sf, layer.features[which_serial_feature]);
strategy.dropped_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
return true; // dropped
}
}
@@ -1640,6 +1640,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
size_t lead_features_count = 0; // of the tile so far
size_t other_multiplier_cluster_features_count = 0; // of the tile so far
bool too_many_features = false;
bool too_many_bytes = false;
std::atomic<bool> within[child_shards];
long long start_geompos[child_shards];
for (size_t i = 0; i < (size_t) child_shards; i++) {
@@ -1741,8 +1744,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
struct multiplier_state multiplier_state;
size_t multiplier_seq = retain_points_multiplier - 1;
bool drop_rest = false; // are we dropping the remainder of a multiplier cluster whose first point was dropped?
bool dropping_by_rate = false; // are we dropping anything by rate in this tile, or keeping it only as part of a multiplier?
bool drop_rest_of_cluster = false; // are we dropping the remainder of a multiplier cluster whose first point was dropped?
bool dropping_by_rate = false; // are we dropping anything by rate in this tile, or keeping it only as part of a multiplier?
unsigned long long next_feature_previndex = 0;
strategy strategy;
@@ -1797,7 +1800,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (sf.dropped == FEATURE_KEPT) {
// this is a new multiplier cluster, so stop dropping features
// that were dropped because the previous lead feature was dropped
drop_rest = false;
drop_rest_of_cluster = false;
} else {
can_stop_early = false;
@@ -1811,7 +1814,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
}
if (sf.dropped == FEATURE_DROPPED || drop_rest) {
if (sf.dropped == FEATURE_DROPPED || drop_rest_of_cluster) {
multiplier_seq = (multiplier_seq + 1) % retain_points_multiplier;
if (find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
@@ -1826,13 +1829,13 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// only the first point of a multiplier cluster can be dropped
// by any of these mechanisms. (but if one is, it drags the whole
// cluster down with it by setting drop_rest).
// cluster down with it by setting drop_rest_of_cluster).
if (sf.dropped == FEATURE_KEPT) {
if (gamma > 0) {
if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.dropped_by_gamma++;
drop_rest = true;
drop_rest_of_cluster = true;
can_stop_early = false;
continue;
}
@@ -1860,14 +1863,14 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.coalesced_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
can_stop_early = false;
continue;
}
} else if (additional[A_DROP_DENSEST_AS_NEEDED]) {
add_sample_to(gaps, sf.gap, gaps_increment, seq);
if (sf.gap < mingap) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest_of_cluster, arg->attribute_accum)) {
can_stop_early = false;
continue;
}
@@ -1892,7 +1895,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.coalesced_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
continue;
}
} else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) {
@@ -1903,7 +1906,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
coalesced_area += sf.extent;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.coalesced_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
can_stop_early = false;
continue;
}
@@ -1912,7 +1915,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// search here is for LLONG_MAX, not minextent, because we are dropping features, not coalescing them,
// so we shouldn't expect to find anything small that we can related this feature to.
if (minextent != 0 && sf.extent + coalesced_area <= minextent) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest_of_cluster, arg->attribute_accum)) {
can_stop_early = false;
continue;
}
@@ -1925,7 +1928,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
coalesced_area += sf.extent;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.coalesced_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
can_stop_early = false;
continue;
}
@@ -1934,7 +1937,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// search here is for LLONG_MAX, not minextent, because we are dropping features, not coalescing them,
// so we shouldn't expect to find anything small that we can related this feature to.
if (mindrop_sequence != 0 && drop_sequence <= mindrop_sequence) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest_of_cluster, arg->attribute_accum)) {
can_stop_early = false;
continue;
}
@@ -1946,7 +1949,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
features[which_serial_feature].coalesced = true;
preserve_attributes(arg->attribute_accum, sf, features[which_serial_feature]);
strategy.coalesced_as_needed++;
drop_rest = true;
drop_rest_of_cluster = true;
can_stop_early = false;
continue;
}
@@ -1995,10 +1998,24 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
unsigned long long sfindex = sf.index;
if (sf.geometry.size() > 0) {
if (lead_features_count > max_tile_size || (lead_features_count + other_multiplier_cluster_features_count > max_tile_features && !prevent[P_FEATURE_LIMIT])) {
// Adjust tile size limit based on the ratio of multiplier cluster features to lead features
size_t scaled_max_tile_size = max_tile_size;
if (lead_features_count > 0) {
scaled_max_tile_size *= (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
}
size_t scaled_max_tile_features = max_tile_features;
if (lead_features_count > 0) {
scaled_max_tile_features *= (lead_features_count + other_multiplier_cluster_features_count) / lead_features_count;
}
if (too_many_bytes || lead_features_count > scaled_max_tile_size) {
// Even being maximally conservative, each feature is still going to be
// at least one byte in the output tile, so this can't possibly work.
skipped++;
too_many_bytes = true;
} else if (too_many_features || (lead_features_count + other_multiplier_cluster_features_count > scaled_max_tile_features && !prevent[P_FEATURE_LIMIT])) {
skipped++;
too_many_features = true;
} else {
kept++;
@@ -2483,13 +2500,15 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
if (totalsize > 0 && tile.layers.size() > 0) {
if (totalsize > scaled_max_tile_features && !prevent[P_FEATURE_LIMIT]) {
if (too_many_features || (totalsize > scaled_max_tile_features && !prevent[P_FEATURE_LIMIT])) {
if (totalsize > arg->feature_count_out) {
arg->feature_count_out = totalsize;
}
size_t estimated = totalsize * (skipped + kept) / kept;
if (!quiet) {
fprintf(stderr, "tile %d/%u/%u has %zu features, >%zu \n", z, tx, ty, totalsize, scaled_max_tile_features);
fprintf(stderr, "tile %d/%u/%u has %zu (estimated %zu) features, >%zu \n", z, tx, ty, totalsize, estimated, scaled_max_tile_features);
}
if (trying_to_stop_early && line_detail == first_detail) {
@@ -2515,7 +2534,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
line_detail++; // to keep it the same when the loop decrements it
continue;
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
mingap_fraction = mingap_fraction * scaled_max_tile_features / totalsize * 0.80;
mingap_fraction = mingap_fraction * scaled_max_tile_features / estimated * 0.80;
unsigned long long m = choose_mingap(gaps, mingap_fraction, mingap);
if (m != mingap) {
mingap = m;
@@ -2530,7 +2549,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
continue;
}
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * scaled_max_tile_features / totalsize * 0.75;
minextent_fraction = minextent_fraction * scaled_max_tile_features / estimated * 0.75;
long long m = choose_minextent(extents, minextent_fraction, minextent);
if (m != minextent) {
minextent = m;
@@ -2548,7 +2567,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// The 95% is a guess to avoid too many retries
// and probably actually varies based on how much duplicated metadata there is
mindrop_sequence_fraction = mindrop_sequence_fraction * scaled_max_tile_features / totalsize * 0.95;
mindrop_sequence_fraction = mindrop_sequence_fraction * scaled_max_tile_features / estimated * 0.95;
unsigned long long m = choose_mindrop_sequence(drop_sequences, mindrop_sequence_fraction, mindrop_sequence);
if (m != mindrop_sequence) {
mindrop_sequence = m;
@@ -2585,7 +2604,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
// printf("%lld %zu\n", estimated_complexity, compressed.size());
}
if (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
if (too_many_bytes || (compressed.size() > scaled_max_tile_size && !prevent[P_KILOBYTE_LIMIT])) {
// Estimate how big it really should have been compressed
// from how many features were kept vs skipped for already being
// over the threshold
@@ -2682,6 +2701,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
exit(EXIT_PTHREAD);
}
if (skipped > 0 || too_many_bytes || too_many_features) {
fprintf(stderr, "Can't happen: writing tile even though we skipped\n");
exit(EXIT_IMPOSSIBLE);
}
if (outdb != NULL) {
mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
} else if (outdir != NULL) {
@@ -2802,7 +2826,6 @@ exit(EXIT_IMPOSSIBLE);
dc.begin();
}
arg->wrote_zoom = z;
long long len;
struct zxy parent(z - 1, x / 2, y / 2);
@@ -2810,6 +2833,7 @@ exit(EXIT_IMPOSSIBLE);
skip_tile(&dc, &geompos, arg->compressed);
len = 1;
} else {
arg->wrote_zoom = z;
len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
}
@@ -2903,6 +2927,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
std::set<zxy> skip_children;
int z;
int largest_written = -1;
for (z = iz; z <= maxzoom; z++) {
std::atomic<long long> most(0);
@@ -3122,6 +3148,9 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
if (args[thread].wrote_zoom > z) {
z = args[thread].wrote_zoom;
}
if (args[thread].wrote_zoom > largest_written) {
largest_written = args[thread].wrote_zoom;
}
if (args[thread].still_dropping) {
extend_zooms = true;
@@ -3184,6 +3213,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
}
}
if (largest_written >= 0 && maxzoom > largest_written) {
maxzoom = largest_written;
}
for (size_t j = 0; j < TEMP_FILES; j++) {
// Can be < 0 if there is only one source file, at z0
if (geomfd[j] >= 0) {