mirror of
https://github.com/felt/tippecanoe.git
synced 2026-11-22 16:31:18 +01:00
Compare commits
21
Commits
main
...
categoricality
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f396dbbb2 | ||
|
|
5c681e50d9 | ||
|
|
82be5627f4 | ||
|
|
c863e4a22c | ||
|
|
ffabc69a45 | ||
|
|
b2681472f8 | ||
|
|
9cd3cdf9da | ||
|
|
80cd39d996 | ||
|
|
b024c515eb | ||
|
|
07ef9f4a3c | ||
|
|
8264b4096a | ||
|
|
fba86ebe24 | ||
|
|
4cd40c72ea | ||
|
|
de607d1440 | ||
|
|
c6379d74d2 | ||
|
|
0c31280339 | ||
|
|
71df08511e | ||
|
|
efa209d175 | ||
|
|
9f97397e21 | ||
|
|
61b5475015 | ||
|
|
3ae800745c |
@@ -2496,6 +2496,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("close sorted index");
|
||||
}
|
||||
|
||||
// provisional merged tilestats, so the stats for each attribute can be used during tiling
|
||||
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
|
||||
|
||||
/* Traverse and split the geometries for each zoom level */
|
||||
|
||||
struct stat geomst;
|
||||
@@ -2518,7 +2521,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
std::atomic<unsigned> midx(0);
|
||||
std::atomic<unsigned> midy(0);
|
||||
std::vector<strategy> strategies;
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies);
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, merged_lm, prefilter, postfilter, attribute_accum, filter, strategies);
|
||||
|
||||
if (maxzoom != written) {
|
||||
if (written > minzoom) {
|
||||
@@ -2573,7 +2576,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
midlon = maxlon;
|
||||
}
|
||||
|
||||
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
|
||||
// merge again, because new entries can have been added during tiling
|
||||
merged_lm = merge_layermaps(layermaps);
|
||||
|
||||
for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) {
|
||||
ai->second.minzoom = minzoom;
|
||||
@@ -2855,6 +2859,7 @@ int main(int argc, char **argv) {
|
||||
{"order-descending-by", required_argument, 0, '~'},
|
||||
{"order-smallest-first", no_argument, 0, '~'},
|
||||
{"order-largest-first", no_argument, 0, '~'},
|
||||
{"order-interesting-first", no_argument, 0, '~'},
|
||||
|
||||
{"Adding calculated attributes", 0, 0, 0},
|
||||
{"calculate-feature-density", no_argument, &additional[A_CALCULATE_FEATURE_DENSITY], 1},
|
||||
@@ -2999,6 +3004,8 @@ int main(int argc, char **argv) {
|
||||
} else if (strcmp(opt, "order-largest-first") == 0) {
|
||||
order_by.push_back(order_field(ORDER_BY_SIZE, true));
|
||||
order_by_size = true;
|
||||
} else if (strcmp(opt, "order-interesting-first") == 0) {
|
||||
order_by.push_back(order_field(ORDER_BY_INTERESTINGNESS, true));
|
||||
} else if (strcmp(opt, "simplification-at-maximum-zoom") == 0) {
|
||||
maxzoom_simplification = atof_require(optarg, "Mazoom simplification");
|
||||
if (maxzoom_simplification <= 0) {
|
||||
|
||||
@@ -69,8 +69,9 @@ struct order_field {
|
||||
extern std::vector<order_field> order_by;
|
||||
|
||||
// not legal UTF-8, so can't appear as a real attribute name
|
||||
#define ORDER_BY_SIZE "\200size"
|
||||
extern bool order_by_size;
|
||||
#define ORDER_BY_SIZE "\200size"
|
||||
#define ORDER_BY_INTERESTINGNESS "\200interestingness"
|
||||
|
||||
int mkstemp_cloexec(char *name);
|
||||
FILE *fopen_oflag(const char *name, const char *mode, int oflag);
|
||||
|
||||
+84
@@ -285,6 +285,18 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
|
||||
state.nospace = true;
|
||||
state.json_write_string(attribute.first);
|
||||
|
||||
double xd = attribute.second.xsum / attribute.second.count;
|
||||
double yd = attribute.second.ysum / attribute.second.count;
|
||||
double d = sqrt(xd * xd + yd * yd);
|
||||
|
||||
#if 0
|
||||
printf("%s %0.6f", attribute.first.c_str(), d);
|
||||
if (attribute.second.numeric_count != 0) {
|
||||
printf(" %f %f", attribute.second.mean, sqrt(attribute.second.m2 / attribute.second.numeric_count));
|
||||
}
|
||||
printf("\n");
|
||||
#endif
|
||||
|
||||
size_t val_count = attribute.second.sample_values.size();
|
||||
if (val_count > max_tilestats_sample_values) {
|
||||
val_count = max_tilestats_sample_values;
|
||||
@@ -840,6 +852,45 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
|
||||
if (fk->second.max > fk2->second.max) {
|
||||
fk2->second.max = fk->second.max;
|
||||
}
|
||||
|
||||
fk2->second.xsum += fk->second.xsum;
|
||||
fk2->second.ysum += fk->second.ysum;
|
||||
fk2->second.count += fk->second.count;
|
||||
|
||||
if (fk->second.numeric_count + fk2->second.numeric_count > 0) {
|
||||
// https://math.stackexchange.com/questions/2971315/how-do-i-combine-standard-deviations-of-two-groups
|
||||
size_t n1 = fk->second.numeric_count;
|
||||
size_t n2 = fk2->second.numeric_count;
|
||||
|
||||
if (n1 == 0) {
|
||||
// fk2 is already correct
|
||||
} else if (n2 == 0) {
|
||||
// move fk to fk2
|
||||
fk2->second.m2 = fk->second.m2;
|
||||
} else {
|
||||
double var_x1 = fk->second.m2 / n1;
|
||||
double var_x2 = fk2->second.m2 / n2;
|
||||
double mean_x1 = fk->second.mean;
|
||||
double mean_x2 = fk2->second.mean;
|
||||
double mean_x = (n1 * mean_x1 + n2 * mean_x2) / (n1 + n2);
|
||||
|
||||
// stack overflow uses n1 - 1 and n2 - 1
|
||||
double q1 = (n1) * var_x1 + n1 * mean_x1 * mean_x1;
|
||||
double q2 = (n2) * var_x2 + n2 * mean_x2 * mean_x2;
|
||||
double qc = q1 + q2;
|
||||
// stack overflow uses n1 + n2 - 1
|
||||
double sc = sqrt((qc - (n1 + n2) * mean_x * mean_x) / (n1 + n2));
|
||||
|
||||
fk2->second.m2 = sc * sc * (n1 + n2);
|
||||
}
|
||||
|
||||
fk2->second.mean = (
|
||||
fk2->second.mean * fk2->second.numeric_count +
|
||||
fk->second.mean * fk->second.numeric_count
|
||||
) / (fk2->second.numeric_count + fk->second.numeric_count);
|
||||
|
||||
fk2->second.numeric_count += fk->second.numeric_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -859,6 +910,20 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
|
||||
return out;
|
||||
}
|
||||
|
||||
double categorical_hash(std::string const &val) {
|
||||
unsigned hash = 0;
|
||||
for (size_t i = 0; i < val.size(); i++) {
|
||||
// https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing
|
||||
hash = hash * 2654435769 + val[i];
|
||||
}
|
||||
// extra multiply so even single-byte values are distributed
|
||||
// around the circle rather than clumped together on one side.
|
||||
hash = hash * 2654435769;
|
||||
double angle = ((double) hash) / UINT_MAX * 2 * M_PI;
|
||||
|
||||
return angle;
|
||||
}
|
||||
|
||||
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &attrib, type_and_string const &val) {
|
||||
if (val.type == mvt_null) {
|
||||
return;
|
||||
@@ -895,5 +960,24 @@ void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, s
|
||||
}
|
||||
}
|
||||
|
||||
double angle = categorical_hash(val.string);
|
||||
double x = cos(angle);
|
||||
double y = sin(angle);
|
||||
|
||||
fka->second.xsum += x;
|
||||
fka->second.ysum += y;
|
||||
fka->second.count++;
|
||||
|
||||
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
|
||||
if (val.type == mvt_double) {
|
||||
double newValue = atof(val.string.c_str());
|
||||
|
||||
fka->second.numeric_count++;
|
||||
double delta = newValue - fka->second.mean;
|
||||
fka->second.mean += delta / fka->second.numeric_count;
|
||||
double delta2 = newValue - fka->second.mean;
|
||||
fka->second.m2 += delta * delta2;
|
||||
}
|
||||
|
||||
fka->second.type |= (1 << val.type);
|
||||
}
|
||||
|
||||
+11
@@ -22,6 +22,16 @@ struct type_and_string_stats {
|
||||
double min = INFINITY;
|
||||
double max = -INFINITY;
|
||||
int type = 0;
|
||||
|
||||
// for circular mean of value hashes
|
||||
double xsum = 0;
|
||||
double ysum = 0;
|
||||
size_t count = 0;
|
||||
|
||||
// for mean and standard deviation
|
||||
double mean = 0;
|
||||
double numeric_count = 0;
|
||||
double m2 = 0;
|
||||
};
|
||||
|
||||
struct layermap_entry {
|
||||
@@ -83,5 +93,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
|
||||
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
|
||||
|
||||
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
|
||||
double categorical_hash(std::string const &value);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "options.hpp"
|
||||
#include "main.hpp"
|
||||
#include "pool.hpp"
|
||||
#include "memfile.hpp"
|
||||
#include "projection.hpp"
|
||||
#include "evaluator.hpp"
|
||||
#include "milo/dtoa_milo.h"
|
||||
@@ -740,6 +741,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
}
|
||||
}
|
||||
|
||||
sf.stringpool = r->poolfile->map;
|
||||
|
||||
long long geomstart = r->geompos;
|
||||
sf.geometry = scaled_geometry;
|
||||
serialize_feature(r->geomfile, &sf, &r->geompos, sst->fname, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y)), false);
|
||||
|
||||
@@ -70,6 +70,7 @@ struct serial_feature {
|
||||
std::vector<serial_val> full_values{};
|
||||
std::string layername = "";
|
||||
bool dropped = false;
|
||||
const char *stringpool; // the pool for this segment, where its keys and values are stored
|
||||
};
|
||||
|
||||
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -63,7 +63,7 @@ struct strategy {
|
||||
|
||||
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
|
||||
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
|
||||
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const std::map<std::string, layermap_entry> &merged_layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
|
||||
|
||||
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user