diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into accumulate-attribute-overzoom-merge

Browse Source
This commit is contained in:
Erica Fischer
2024-01-23 14:36:28 -08:00
parent 07ebc0f131 f957f30f90
commit 134ca32eaf
8 changed files with 108 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+2 -2
View File
@@ -480,10 +480,10 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
sst.resize(CPUS); sst.resize(CPUS);
pthread_t pthreads[CPUS]; pthread_t pthreads[CPUS];
std::vector<std::set<type_and_string> > file_subkeys; std::vector<std::set<serial_val> > file_subkeys;
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
file_subkeys.push_back(std::set<type_and_string>()); file_subkeys.push_back(std::set<serial_val>());
} }
for (size_t i = 0; i < CPUS; i++) { for (size_t i = 0; i < CPUS; i++) {
mbtiles.cpp
+49 -49
View File
@@ -196,21 +196,21 @@ void mbtiles_erase_zoom(sqlite3 *outdb, int z) {
} }
} }
bool type_and_string::operator<(const type_and_string &o) const { bool serial_val::operator<(const serial_val &o) const {
if (string < o.string) { if (s < o.s) {
return true; return true;
} }
if (string == o.string && type < o.type) { if (s == o.s && type < o.type) {
return true; return true;
} }
return false; return false;
} }
bool type_and_string::operator!=(const type_and_string &o) const { bool serial_val::operator!=(const serial_val &o) const {
if (type != o.type) { if (type != o.type) {
return true; return true;
} }
if (string != o.string) { if (s != o.s) {
return true; return true;
} }
return false; return false;
@@ -261,7 +261,7 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
state.nospace = true; state.nospace = true;
state.json_write_string(geomtype); state.json_write_string(geomtype);
size_t attrib_count = layer.second.file_keys.size(); size_t attrib_count = layer.second.tilestats.size();
if (attrib_count > max_tilestats_attributes) { if (attrib_count > max_tilestats_attributes) {
attrib_count = max_tilestats_attributes; attrib_count = max_tilestats_attributes;
} }
@@ -277,7 +277,7 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
state.json_write_array(); state.json_write_array();
size_t attrs = 0; size_t attrs = 0;
for (auto attribute : layer.second.file_keys) { for (auto attribute : layer.second.tilestats) {
if (attrs == elements) { if (attrs == elements) {
break; break;
} }
@@ -338,15 +338,15 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
vals++; vals++;
state.nospace = true; state.nospace = true;
state.json_write_stringified(value.string); state.json_write_stringified(value.s);
} else { } else {
std::string trunc = truncate16(value.string, 256); std::string trunc = truncate16(value.s, 256);
if (trunc.size() == value.string.size()) { if (trunc.size() == value.s.size()) {
vals++; vals++;
state.nospace = true; state.nospace = true;
state.json_write_string(value.string); state.json_write_string(value.s);
} }
} }
} }
@@ -714,7 +714,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
} }
for (size_t i = 0; i < lnames.size(); i++) { for (size_t i = 0; i < lnames.size(); i++) {
auto fk = layermap.find(lnames[i]); auto ts = layermap.find(lnames[i]);
state.nospace = true; state.nospace = true;
state.json_write_hash(); state.json_write_hash();
@@ -726,17 +726,17 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
state.nospace = true; state.nospace = true;
state.json_write_string("description"); state.json_write_string("description");
state.nospace = true; state.nospace = true;
state.json_write_string(fk->second.description); state.json_write_string(ts->second.description);
state.nospace = true; state.nospace = true;
state.json_write_string("minzoom"); state.json_write_string("minzoom");
state.nospace = true; state.nospace = true;
state.json_write_signed(fk->second.minzoom); state.json_write_signed(ts->second.minzoom);
state.nospace = true; state.nospace = true;
state.json_write_string("maxzoom"); state.json_write_string("maxzoom");
state.nospace = true; state.nospace = true;
state.json_write_signed(fk->second.maxzoom); state.json_write_signed(ts->second.maxzoom);
state.nospace = true; state.nospace = true;
state.json_write_string("fields"); state.json_write_string("fields");
@@ -745,7 +745,7 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
bool first = true; bool first = true;
size_t attribute_count = 0; size_t attribute_count = 0;
for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) { for (auto j = ts->second.tilestats.begin(); j != ts->second.tilestats.end(); ++j) {
if (first) { if (first) {
first = false; first = false;
} }
@@ -852,35 +852,35 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) { for (auto ts = map->second.tilestats.begin(); ts != map->second.tilestats.end(); ++ts) {
std::string attribname = fk->first; std::string attribname = ts->first;
if (trunc) { if (trunc) {
attribname = truncate16(attribname, 256); attribname = truncate16(attribname, 256);
} }
auto fk2 = out_entry->second.file_keys.find(attribname); auto ts2 = out_entry->second.tilestats.find(attribname);
if (fk2 == out_entry->second.file_keys.end()) { if (ts2 == out_entry->second.tilestats.end()) {
out_entry->second.file_keys.insert(std::pair<std::string, type_and_string_stats>(attribname, fk->second)); out_entry->second.tilestats.insert(std::pair<std::string, tilestat>(attribname, ts->second));
} else { } else {
for (auto val : fk->second.sample_values) { for (auto val : ts->second.sample_values) {
auto pt = std::lower_bound(fk2->second.sample_values.begin(), fk2->second.sample_values.end(), val); auto pt = std::lower_bound(ts2->second.sample_values.begin(), ts2->second.sample_values.end(), val);
if (pt == fk2->second.sample_values.end() || *pt != val) { // not found if (pt == ts2->second.sample_values.end() || *pt != val) { // not found
fk2->second.sample_values.insert(pt, val); ts2->second.sample_values.insert(pt, val);
if (fk2->second.sample_values.size() > max_tilestats_sample_values) { if (ts2->second.sample_values.size() > max_tilestats_sample_values) {
fk2->second.sample_values.pop_back(); ts2->second.sample_values.pop_back();
} }
} }
} }
fk2->second.type |= fk->second.type; ts2->second.type |= ts->second.type;
if (fk->second.min < fk2->second.min) { if (ts->second.min < ts2->second.min) {
fk2->second.min = fk->second.min; ts2->second.min = ts->second.min;
} }
if (fk->second.max > fk2->second.max) { if (ts->second.max > ts2->second.max) {
fk2->second.max = fk->second.max; ts2->second.max = ts->second.max;
} }
} }
} }
@@ -901,41 +901,41 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
return out; return out;
} }
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &attrib, type_and_string const &val) { void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &attrib, serial_val const &val) {
if (val.type == mvt_null) { if (val.type == mvt_null) {
return; return;
} }
auto fka = file_keys.find(attrib); auto tsa = tilestats.find(attrib);
if (fka == file_keys.end()) { if (tsa == tilestats.end()) {
file_keys.insert(std::pair<std::string, type_and_string_stats>(attrib, type_and_string_stats())); tilestats.insert(std::pair<std::string, tilestat>(attrib, tilestat()));
fka = file_keys.find(attrib); tsa = tilestats.find(attrib);
} }
if (fka == file_keys.end()) { if (tsa == tilestats.end()) {
fprintf(stderr, "Can't happen (tilestats)\n"); fprintf(stderr, "Can't happen (tilestats)\n");
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
if (val.type == mvt_double) { if (val.type == mvt_double) {
double d = atof(val.string.c_str()); double d = atof(val.s.c_str());
if (d < fka->second.min) { if (d < tsa->second.min) {
fka->second.min = d; tsa->second.min = d;
} }
if (d > fka->second.max) { if (d > tsa->second.max) {
fka->second.max = d; tsa->second.max = d;
} }
} }
auto pt = std::lower_bound(fka->second.sample_values.begin(), fka->second.sample_values.end(), val); auto pt = std::lower_bound(tsa->second.sample_values.begin(), tsa->second.sample_values.end(), val);
if (pt == fka->second.sample_values.end() || *pt != val) { // not found if (pt == tsa->second.sample_values.end() || *pt != val) { // not found
fka->second.sample_values.insert(pt, val); tsa->second.sample_values.insert(pt, val);
if (fka->second.sample_values.size() > max_tilestats_sample_values) { if (tsa->second.sample_values.size() > max_tilestats_sample_values) {
fka->second.sample_values.pop_back(); tsa->second.sample_values.pop_back();
} }
} }
fka->second.type |= (1 << val.type); tsa->second.type |= (1 << val.type);
} }
mbtiles.hpp
+4 -12
View File
@@ -9,16 +9,8 @@ extern size_t max_tilestats_attributes;
extern size_t max_tilestats_sample_values; extern size_t max_tilestats_sample_values;
extern size_t max_tilestats_values; extern size_t max_tilestats_values;
struct type_and_string { struct tilestat {
int type = 0; std::vector<serial_val> sample_values = std::vector<serial_val>(); // sorted
std::string string = "";
bool operator<(const type_and_string &o) const;
bool operator!=(const type_and_string &o) const;
};
struct type_and_string_stats {
std::vector<type_and_string> sample_values = std::vector<type_and_string>(); // sorted
double min = INFINITY; double min = INFINITY;
double max = -INFINITY; double max = -INFINITY;
int type = 0; int type = 0;
@@ -26,7 +18,7 @@ struct type_and_string_stats {
struct layermap_entry { struct layermap_entry {
size_t id = 0; size_t id = 0;
std::map<std::string, type_and_string_stats> file_keys{}; std::map<std::string, tilestat> tilestats{};
int minzoom = 0; int minzoom = 0;
int maxzoom = 0; int maxzoom = 0;
std::string description = ""; std::string description = "";
@@ -84,7 +76,7 @@ void mbtiles_close(sqlite3 *outdb, const char *pgm);
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps); std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps);
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc); std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val); void add_to_tilestats(std::map<std::string, tilestat> &tilestats, std::string const &layername, serial_val const &val);
unsigned long long fnv1a(std::string const &s); unsigned long long fnv1a(std::string const &s);
plugin.cpp
+25 -25
View File
@@ -241,24 +241,24 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
} }
} }
auto fk = layermap.find(layername); auto ts = layermap.find(layername);
if (fk == layermap.end()) { if (ts == layermap.end()) {
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
if (z < fk->second.minzoom) { if (z < ts->second.minzoom) {
fk->second.minzoom = z; ts->second.minzoom = z;
} }
if (z > fk->second.maxzoom) { if (z > ts->second.maxzoom) {
fk->second.maxzoom = z; ts->second.maxzoom = z;
} }
if (feature.type == mvt_point) { if (feature.type == mvt_point) {
fk->second.points++; ts->second.points++;
} else if (feature.type == mvt_linestring) { } else if (feature.type == mvt_linestring) {
fk->second.lines++; ts->second.lines++;
} else if (feature.type == mvt_polygon) { } else if (feature.type == mvt_polygon) {
fk->second.polygons++; ts->second.polygons++;
} }
for (size_t i = 0; i < properties->value.object.length; i++) { for (size_t i = 0; i < properties->value.object.length; i++) {
@@ -274,11 +274,11 @@ std::vector<mvt_layer> parse_layers(int fd, int z, unsigned x, unsigned y, std::
mvt_value v = stringified_to_mvt_value(tp, s.c_str()); mvt_value v = stringified_to_mvt_value(tp, s.c_str());
l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v); l->second.tag(feature, std::string(properties->value.object.keys[i]->value.string.string), v);
type_and_string attrib; serial_val attrib;
attrib.type = tp; attrib.type = tp;
attrib.string = s; attrib.s = s;
add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib); add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib);
} }
} }
@@ -478,27 +478,27 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
} }
} }
auto fk = layermap.find(layername); auto ts = layermap.find(layername);
if (fk == layermap.end()) { if (ts == layermap.end()) {
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
sf.layer = fk->second.id; sf.layer = ts->second.id;
if (z < fk->second.minzoom) { if (z < ts->second.minzoom) {
fk->second.minzoom = z; ts->second.minzoom = z;
} }
if (z > fk->second.maxzoom) { if (z > ts->second.maxzoom) {
fk->second.maxzoom = z; ts->second.maxzoom = z;
} }
if (!postfilter) { if (!postfilter) {
if (sf.t == mvt_point) { if (sf.t == mvt_point) {
fk->second.points++; ts->second.points++;
} else if (sf.t == mvt_linestring) { } else if (sf.t == mvt_linestring) {
fk->second.lines++; ts->second.lines++;
} else if (sf.t == mvt_polygon) { } else if (sf.t == mvt_polygon) {
fk->second.polygons++; ts->second.polygons++;
} }
} }
@@ -515,12 +515,12 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string)); sf.full_keys.push_back(std::string(properties->value.object.keys[i]->value.string.string));
sf.full_values.push_back(v); sf.full_values.push_back(v);
type_and_string attrib; serial_val attrib;
attrib.string = v.s; attrib.s = v.s;
attrib.type = v.type; attrib.type = v.type;
if (!postfilter) { if (!postfilter) {
add_to_file_keys(fk->second.file_keys, std::string(properties->value.object.keys[i]->value.string.string), attrib); add_to_tilestats(ts->second.tilestats, std::string(properties->value.object.keys[i]->value.string.string), attrib);
} }
} }
} }
serial.cpp
+2 -6
View File
@@ -827,12 +827,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
if (!sst->filters) { if (!sst->filters) {
for (size_t i = 0; i < sf.full_keys.size(); i++) { for (size_t i = 0; i < sf.full_keys.size(); i++) {
type_and_string attrib; auto ts = sst->layermap->find(sf.layername);
attrib.type = sf.full_values[i].type; add_to_tilestats(ts->second.tilestats, sf.full_keys[i], sf.full_values[i]);
attrib.string = sf.full_values[i].s;
auto fk = sst->layermap->find(sf.layername);
add_to_file_keys(fk->second.file_keys, sf.full_keys[i], attrib);
} }
} }
serial.hpp
+3
View File
@@ -38,6 +38,9 @@ void deserialize_byte(char **f, signed char *n);
struct serial_val { struct serial_val {
int type = 0; int type = 0;
std::string s = ""; std::string s = "";
bool operator<(const serial_val &o) const;
bool operator!=(const serial_val &o) const;
}; };
struct serial_feature { struct serial_feature {
tile-join.cpp
+20 -24
View File
@@ -134,7 +134,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
} }
} }
auto file_keys = layermap.find(layer.name); auto tilestats = layermap.find(layer.name);
for (size_t f = 0; f < layer.features.size(); f++) { for (size_t f = 0; f < layer.features.size(); f++) {
mvt_feature feat = layer.features[f]; mvt_feature feat = layer.features[f];
@@ -152,7 +152,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
outfeature.id = feat.id; outfeature.id = feat.id;
} }
std::map<std::string, std::pair<mvt_value, type_and_string>> attributes; std::map<std::string, std::pair<mvt_value, serial_val>> attributes;
std::vector<std::string> key_order; std::vector<std::string> key_order;
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
@@ -160,16 +160,12 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
mvt_value &val = layer.values[feat.tags[t + 1]]; mvt_value &val = layer.values[feat.tags[t + 1]];
serial_val sv = mvt_value_to_serial_val(val); serial_val sv = mvt_value_to_serial_val(val);
if (sv.type < 0) { if (sv.type == mvt_null) {
continue; continue;
} }
if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) { if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) {
type_and_string tas; attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(key, std::pair<mvt_value, serial_val>(val, sv)));
tas.type = sv.type;
tas.string = sv.s;
attributes.insert(std::pair<std::string, std::pair<mvt_value, type_and_string>>(key, std::pair<mvt_value, type_and_string>(val, tas)));
key_order.push_back(key); key_order.push_back(key);
} }
@@ -212,14 +208,14 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
attributes.erase(fa); attributes.erase(fa);
} }
type_and_string tas; serial_val outsv;
tas.type = outval.type; outsv.type = outval.type;
tas.string = joinval; outsv.s = joinval;
// Convert from double to int if the joined attribute is an integer // Convert from double to int if the joined attribute is an integer
outval = stringified_to_mvt_value(outval.type, joinval.c_str()); outval = stringified_to_mvt_value(outval.type, joinval.c_str());
attributes.insert(std::pair<std::string, std::pair<mvt_value, type_and_string>>(joinkey, std::pair<mvt_value, type_and_string>(outval, tas))); attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(joinkey, std::pair<mvt_value, serial_val>(outval, outsv)));
key_order.push_back(joinkey); key_order.push_back(joinkey);
} }
} }
@@ -228,11 +224,11 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
} }
if (matched || !ifmatched) { if (matched || !ifmatched) {
if (file_keys == layermap.end()) { if (tilestats == layermap.end()) {
layermap.insert(std::pair<std::string, layermap_entry>(layer.name, layermap_entry(layermap.size()))); layermap.insert(std::pair<std::string, layermap_entry>(layer.name, layermap_entry(layermap.size())));
file_keys = layermap.find(layer.name); tilestats = layermap.find(layer.name);
file_keys->second.minzoom = z; tilestats->second.minzoom = z;
file_keys->second.maxzoom = z; tilestats->second.maxzoom = z;
} }
// To keep attributes in their original order instead of alphabetical // To keep attributes in their original order instead of alphabetical
@@ -241,7 +237,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
if (fa != attributes.end()) { if (fa != attributes.end()) {
outlayer.tag(outfeature, k, fa->second.first); outlayer.tag(outfeature, k, fa->second.first);
add_to_file_keys(file_keys->second.file_keys, k, fa->second.second); add_to_tilestats(tilestats->second.tilestats, k, fa->second.second);
attributes.erase(fa); attributes.erase(fa);
} }
} }
@@ -259,19 +255,19 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
features_added++; features_added++;
outlayer.features.push_back(outfeature); outlayer.features.push_back(outfeature);
if (z < file_keys->second.minzoom) { if (z < tilestats->second.minzoom) {
file_keys->second.minzoom = z; tilestats->second.minzoom = z;
} }
if (z > file_keys->second.maxzoom) { if (z > tilestats->second.maxzoom) {
file_keys->second.maxzoom = z; tilestats->second.maxzoom = z;
} }
if (feat.type == mvt_point) { if (feat.type == mvt_point) {
file_keys->second.points++; tilestats->second.points++;
} else if (feat.type == mvt_linestring) { } else if (feat.type == mvt_linestring) {
file_keys->second.lines++; tilestats->second.lines++;
} else if (feat.type == mvt_polygon) { } else if (feat.type == mvt_polygon) {
file_keys->second.polygons++; tilestats->second.polygons++;
} }
} }
} }
tile.cpp
+3 -7
View File
@@ -1812,17 +1812,13 @@ void add_tilestats(std::string const &layername, int z, std::vector<std::map<std
(*layer_unmaps)[tiling_seg][lme.id] = layername; (*layer_unmaps)[tiling_seg][lme.id] = layername;
} }
} }
auto fk = layermap.find(layername); auto ts = layermap.find(layername);
if (fk == layermap.end()) { if (ts == layermap.end()) {
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str()); fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
exit(EXIT_IMPOSSIBLE); exit(EXIT_IMPOSSIBLE);
} }
type_and_string attrib; add_to_tilestats(ts->second.tilestats, key, val);
attrib.type = val.type;
attrib.string = val.s;
add_to_file_keys(fk->second.file_keys, key, attrib);
} }
void promote_attribute(std::string const &key, partial &p, char *stringpool, long long *pool_off) { void promote_attribute(std::string const &key, partial &p, char *stringpool, long long *pool_off) {