diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Make the numeric accumulation prefix configurable

Browse Source
This commit is contained in:
Erica Fischer
2024-09-18 10:36:09 -07:00
parent 21bf44720d
commit 59026112f6
9 changed files with 52 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+6 -6
View File
@@ -367,15 +367,15 @@ overzoom-test: tippecanoe-overzoom
cmp tests/pbf/bin-11-327-791.pbf.out.json.check tests/pbf/bin-11-327-791.pbf.out.json
rm tests/pbf/bin-11-327-791.pbf.out.json.check tests/pbf/bin-11-327-791.pbf.out
# Binning with longitude wraparound problems
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-2-0-1.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-2-0-1.geojson tests/pbf/0-0-0.pbf 2/0/1 2/0/1
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-2-0-1.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-2-0-1.geojson tests/pbf/0-0-0.pbf 2/0/1 2/0/1
./tippecanoe-decode tests/pbf/0-0-0-pop-2-0-1.pbf.out 2 0 1 > tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check
cmp tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check tests/pbf/0-0-0-pop-2-0-1.pbf.out.json
rm tests/pbf/0-0-0-pop-2-0-1.pbf.out tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-1-1-0.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-1-1-0.geojson tests/pbf/0-0-0.pbf 1/1/0 1/1/0
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-1-1-0.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-1-1-0.geojson tests/pbf/0-0-0.pbf 1/1/0 1/1/0
./tippecanoe-decode tests/pbf/0-0-0-pop-1-1-0.pbf.out 1 1 0 > tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check
cmp tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check tests/pbf/0-0-0-pop-1-1-0.pbf.out.json
rm tests/pbf/0-0-0-pop-1-1-0.pbf.out tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-0-0-0.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-0-0-0.geojson tests/pbf/0-0-0.pbf 0/0/0 0/0/0
./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-0-0-0.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-0-0-0.geojson tests/pbf/0-0-0.pbf 0/0/0 0/0/0
./tippecanoe-decode tests/pbf/0-0-0-pop-0-0-0.pbf.out 0 0 0 > tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check
cmp tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check tests/pbf/0-0-0-pop-0-0-0.pbf.out.json
rm tests/pbf/0-0-0-pop-0-0-0.pbf.out tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check
@@ -520,7 +520,7 @@ accumulate-test:
test `grep '"POP1950": [0-9]' tests/ne_110m_populated_places_nulls/in.json | wc -l` == 144
# and 99 without it
test `grep '"POP1950": null' tests/ne_110m_populated_places_nulls/in.json | wc -l` == 99
./tippecanoe -yNAME -yPOP1950 -q -z3 -r1.75 -b0 -f -e tests/pbf/accum.dir --accumulate-numeric-attributes --set-attribute clustersize:1 --accumulate-attribute clustersize:sum --retain-points-multiplier 3 tests/ne_110m_populated_places_nulls/in.json
./tippecanoe -yNAME -yPOP1950 -q -z3 -r1.75 -b0 -f -e tests/pbf/accum.dir --accumulate-numeric-attributes=tippecanoe --set-attribute clustersize:1 --accumulate-attribute clustersize:sum --retain-points-multiplier 3 tests/ne_110m_populated_places_nulls/in.json
# at this drop rate, there are 6 points at z0 that have no POP1950s clustered onto them....
test `./tippecanoe-decode -c tests/pbf/accum.dir/0/0/0.pbf 0 0 0 | grep -v 'tippecanoe:count:POP1950' | wc -l` == 78
# 35 of which have no POP1950 at all
@@ -544,7 +544,7 @@ accumulate-test:
# which is the correct 161590
#
# OK, so do these still hold after megatile filtering?
./tippecanoe-overzoom --accumulate-numeric-attributes --accumulate-attribute clustersize:sum -m -o tests/pbf/accum-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0
./tippecanoe-overzoom --accumulate-numeric-attributes=tippecanoe --accumulate-attribute clustersize:sum -m -o tests/pbf/accum-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0
# Now there are 40 features with POP1950 clusters
test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep 'tippecanoe:count:POP1950' | wc -l` == 40
# There are 4 with bare POP1950
@@ -560,7 +560,7 @@ accumulate-test:
# which add up to 161590 so we have the right global total
#
# Now on to binning!
./tippecanoe-overzoom --assign-to-bins tests/pbf/h3-0-0-0.geojson --accumulate-numeric-attributes --accumulate-attribute clustersize:sum -m -o tests/pbf/bins-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0
./tippecanoe-overzoom --assign-to-bins tests/pbf/h3-0-0-0.geojson --accumulate-numeric-attributes=tippecanoe --accumulate-attribute clustersize:sum -m -o tests/pbf/bins-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0
# Now there are 30 bins with POP1950 clusters
echo test `./tippecanoe-decode -c tests/pbf/bins-0-0-0.pbf 0 0 0 | grep 'tippecanoe:count:POP1950' | wc -l` == 30
# There are none with bare POP1950 (which is expected; we should only have summary statistics)
clip.cpp
+31 -20
View File
@@ -1082,7 +1082,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data, double simplification,
double tiny_polygon_size, std::vector<mvt_layer> const &bins,
bool accumulate_numeric) {
std::string const &accumulate_numeric) {
std::vector<source_tile> decoded;
for (auto const &t : tiles) {
@@ -1122,20 +1122,24 @@ struct tile_feature {
size_t seq = 0;
};
static void add_mean(mvt_feature &feature, mvt_layer &layer) {
static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const &accumulate_numeric) {
std::string accumulate_numeric_colon = accumulate_numeric + ":";
std::unordered_map<std::string, size_t> attributes;
for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
std::string const &key = layer.keys[feature.tags[i]];
if (starts_with(key, "tippecanoe:")) {
if (starts_with(key, accumulate_numeric_colon)) {
attributes.emplace(key, i);
}
}
for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
std::string accumulate_numeric_sum_colon = accumulate_numeric + ":sum:";
std::string const &key = layer.keys[feature.tags[i]];
if (starts_with(key, "tippecanoe:sum:")) {
std::string trunc = key.substr(15);
auto const f = attributes.find("tippecanoe:count:" + trunc);
if (starts_with(key, accumulate_numeric_sum_colon)) {
std::string trunc = key.substr(accumulate_numeric_sum_colon.size());
auto const f = attributes.find(accumulate_numeric + ":count:" + trunc);
if (f != attributes.end()) {
mvt_value const &sum = layer.values[feature.tags[i + 1]];
mvt_value const &count = layer.values[feature.tags[f->second + 1]];
@@ -1147,7 +1151,7 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer) {
mvt_value mean;
mean.type = mvt_double;
mean.numeric_value.double_value = mvt_value_to_double(sum) / count_val;
layer.tag(feature, "tippecanoe:mean:" + trunc, mean);
layer.tag(feature, accumulate_numeric + ":mean:" + trunc, mean);
}
}
}
@@ -1157,7 +1161,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
std::set<std::string> const &keep,
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::shared_ptr<std::string> const &tile_stringpool,
bool accumulate_numeric) {
std::string const &accumulate_numeric) {
// Add geometry to output feature
mvt_feature outfeature;
@@ -1176,7 +1180,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
outfeature.seq = features[0].seq;
if (attribute_accum.size() > 0 || accumulate_numeric) {
if (attribute_accum.size() > 0 || accumulate_numeric.size() > 0) {
// convert the attributes of the output feature
// from mvt_value to serial_val so they can have
// attributes from the other features of the
@@ -1194,7 +1198,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
// this attribute has an accumulator, so convert it
full_keys.push_back(features[0].layer->keys[features[0].tags[i]]);
full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]]));
} else if (accumulate_numeric && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) {
} else if (accumulate_numeric.size() > 0 && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) {
// convert numeric for accumulation
numeric_out_field.emplace(key, full_keys.size());
full_keys.push_back(key);
@@ -1226,14 +1230,14 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
if (f != attribute_accum.end()) {
serial_val val = mvt_value_to_serial_val(features[i].layer->values[features[i].tags[j + 1]]);
preserve_attribute(f->second, key, val, full_keys, full_values, attribute_accum_state);
} else if (accumulate_numeric) {
} else if (accumulate_numeric.size() > 0) {
const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]];
if (val.is_numeric()) {
// If this is a numeric attribute, but there is also a tippecanoe:sum (etc.) for the
// same attribute, we want to use that one instead of this one.
for (auto const &op : numeric_operations) {
std::string compound_key = "tippecanoe:" + op.first + ":" + key;
std::string compound_key = accumulate_numeric + ":" + op.first + ":" + key;
auto compound_found = keys.find(compound_key);
if (compound_found != keys.end()) {
// found, so skip this one
@@ -1245,8 +1249,8 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
// it is the wrong one.
std::string outkey = key;
if (starts_with(outkey, "tippecanoe:")) {
std::string prefix = "tippecanoe:" + op.first + ":";
if (starts_with(outkey, accumulate_numeric + ":")) {
std::string prefix = accumulate_numeric + ":" + op.first + ":";
if (starts_with(outkey, prefix)) {
outkey = outkey.substr(prefix.size());
} else {
@@ -1254,7 +1258,7 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
}
}
// and then put it back on for the output field
std::string prefixed = "tippecanoe:" + op.first + ":" + outkey;
std::string prefixed = accumulate_numeric + ":" + op.first + ":" + outkey;
// Does it exist in the output feature already?
@@ -1310,8 +1314,8 @@ static void feature_out(std::vector<tile_feature> const &features, mvt_layer &ou
}
}
if (accumulate_numeric) {
add_mean(outfeature, outlayer);
if (accumulate_numeric.size() > 0) {
add_mean(outfeature, outlayer, accumulate_numeric);
}
} else {
for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) {
@@ -1433,7 +1437,7 @@ static bool bbox_intersects(long long x1min, long long y1min, long long x1max, l
}
mvt_tile assign_to_bins(mvt_tile const &features, std::vector<mvt_layer> const &bins, int z, int x, int y, int detail,
bool accumulate_numeric) {
std::string const &accumulate_numeric) {
std::vector<index_event> events;
// Index bins
@@ -1537,7 +1541,14 @@ mvt_tile assign_to_bins(mvt_tile const &features, std::vector<mvt_layer> const &
mvt_value val;
val.type = mvt_uint;
val.numeric_value.uint_value = outfeatures[found->outfeature].size() - 1;
outlayer.tag(nfeature, "tippecanoe:count", val);
std::string attrname;
if (accumulate_numeric.size() == 0) {
attrname = "tippecanoe:count";
} else {
attrname = accumulate_numeric + ":count";
}
outlayer.tag(nfeature, attrname, val);
}
active.erase(found);
@@ -1560,7 +1571,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data, double simplification,
double tiny_polygon_size, std::vector<mvt_layer> const &bins,
bool accumulate_numeric) {
std::string const &accumulate_numeric) {
mvt_tile outtile;
std::shared_ptr<std::string> tile_stringpool = std::make_shared<std::string>();
geometry.hpp
+2 -2
View File
@@ -122,7 +122,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data, double simplification,
double tiny_polygon_size, std::vector<mvt_layer> const &bins,
bool accumulate_numeric);
std::string const &accumulate_numeric);
std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
@@ -131,7 +131,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
std::unordered_map<std::string, attribute_op> const &attribute_accum,
std::vector<std::string> const &unidecode_data, double simplification,
double tiny_polygon_size, std::vector<mvt_layer> const &bins,
bool accumulate_numeric);
std::string const &accumulate_numeric);
draw center_of_mass_mp(const drawvec &dv);
main.cpp
+4 -1
View File
@@ -97,6 +97,7 @@ long long extend_zooms_max = 0;
int retain_points_multiplier = 1;
std::vector<std::string> unidecode_data;
size_t maximum_string_attribute_length = 0;
std::string accumulate_numeric;
std::vector<order_field> order_by;
bool order_reverse;
@@ -3084,7 +3085,7 @@ int main(int argc, char **argv) {
{"attribute-type", required_argument, 0, 'T'},
{"attribute-description", required_argument, 0, 'Y'},
{"accumulate-attribute", required_argument, 0, 'E'},
{"accumulate-numeric-attributes", no_argument, &additional[A_ACCUMULATE_NUMERIC], 1},
{"accumulate-numeric-attributes", required_argument, 0, '~'},
{"empty-csv-columns-are-null", no_argument, &prevent[P_EMPTY_CSV_COLUMNS], 1},
{"convert-stringified-ids-to-numbers", no_argument, &additional[A_CONVERT_NUMERIC_IDS], 1},
{"use-attribute-for-id", required_argument, 0, '~'},
@@ -3327,6 +3328,8 @@ int main(int argc, char **argv) {
unidecode_data = read_unidecode(optarg);
} else if (strcmp(opt, "maximum-string-attribute-length") == 0) {
maximum_string_attribute_length = atoll_require(optarg, "Maximum string attribute length");
} else if (strcmp(opt, "accumulate-numeric-attributes") == 0) {
accumulate_numeric = optarg;
} else {
fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
exit(EXIT_ARGS);
main.hpp
+1
View File
@@ -64,6 +64,7 @@ extern std::map<std::string, serial_val> set_attributes;
extern long long extend_zooms_max;
extern int retain_points_multiplier;
extern size_t maximum_string_attribute_length;
extern std::string accumulate_numeric;
struct order_field {
std::string name;
options.hpp
-1
View File
@@ -12,7 +12,6 @@
#define A_PREFER_RADIX_SORT ((int) 'R')
#define A_COALESCE_DENSEST_AS_NEEDED ((int) 'S')
#define A_CALCULATE_INDEX ((int) 'X')
#define A_ACCUMULATE_NUMERIC ((int) 'a')
#define A_DETECT_SHARED_BORDERS ((int) 'b')
#define A_COALESCE ((int) 'c')
#define A_DROP_FRACTION_AS_NEEDED ((int) 'd')
overzoom.cpp
+3 -3
View File
@@ -22,7 +22,7 @@ bool preserve_input_order = false;
std::unordered_map<std::string, attribute_op> attribute_accum;
std::vector<std::string> unidecode_data;
std::vector<mvt_layer> bins;
bool accumulate_numeric = false;
std::string accumulate_numeric;
std::set<std::string> keep;
@@ -58,7 +58,7 @@ int main(int argc, char **argv) {
{"tiny-polygon-size", required_argument, 0, 's' & 0x1F},
{"source-tile", required_argument, 0, 't'},
{"assign-to-bins", required_argument, 0, 'b' & 0x1F},
{"accumulate-numeric-attributes", no_argument, 0, 'a' & 0x1F},
{"accumulate-numeric-attributes", required_argument, 0, 'a' & 0x1F},
{0, 0, 0, 0},
};
@@ -130,7 +130,7 @@ int main(int argc, char **argv) {
break;
case 'a' & 0x1F:
accumulate_numeric = true;
accumulate_numeric = optarg;
break;
default:
tile-join.cpp
+1 -1
View File
@@ -712,7 +712,7 @@ struct tileset_reader {
t.y = parent_tile.y;
tv.push_back(std::move(t));
std::string ret = overzoom(tv, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map<std::string, attribute_op>(), unidecode_data, 0, 0, std::vector<mvt_layer>(), false);
std::string ret = overzoom(tv, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map<std::string, attribute_op>(), unidecode_data, 0, 0, std::vector<mvt_layer>(), "");
return ret;
}
tile.cpp
+4 -4
View File
@@ -1472,13 +1472,13 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
promote_attribute(key, p);
preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state);
} else if (type == mvt_double && additional[A_ACCUMULATE_NUMERIC]) {
} else if (type == mvt_double && accumulate_numeric.size() > 0) {
for (auto const &operation : numeric_operations) {
serial_val sv;
sv.type = sf.stringpool[sf.values[i]];
sv.s = sf.stringpool + sf.values[i] + 1;
std::string prefixed_key = "tippecanoe:" + operation.first + ":" + key;
std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key;
promote_attribute_prefix(key, prefixed_key, p);
preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state);
}
@@ -1494,9 +1494,9 @@ void preserve_attributes(std::unordered_map<std::string, attribute_op> const *at
promote_attribute(key, p); // promotes it in the target feature
preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state);
} else if (type == mvt_double && additional[A_ACCUMULATE_NUMERIC]) {
} else if (type == mvt_double && accumulate_numeric.size() > 0) {
for (auto const &operation : numeric_operations) {
std::string prefixed_key = "tippecanoe:" + operation.first + ":" + key;
std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key;
promote_attribute_prefix(key, prefixed_key, p);
preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state);
}