Compare commits

...
21 Commits
Author SHA1 Message Date
Erica Fischer 1f396dbbb2 Do some feature ranking 2023-02-10 13:21:30 -08:00
Erica Fischer 5c681e50d9 Weight outlieriness by categoricality 2023-02-10 12:21:20 -08:00
Erica Fischer 82be5627f4 Plumb the layer name through, so we can find the layer stats 2023-02-10 11:18:36 -08:00
Erica Fischer c863e4a22c Pass the layermaps into the feature ordering function 2023-02-09 16:29:34 -08:00
Erica Fischer ffabc69a45 Oh right, remove this duplication too 2023-02-09 15:50:11 -08:00
Erica Fischer b2681472f8 Remove duplicated code 2023-02-09 13:51:53 -08:00
Erica Fischer 9cd3cdf9da Add a string pool pointer to the feature; stop passing it as an arg 2023-02-09 13:47:00 -08:00
Erica Fischer 80cd39d996 Pass the merged layermaps into tiling 2023-02-09 13:06:44 -08:00
Erica Fischer b024c515eb Multithreaded standard deviation is consistent now 2023-02-09 12:14:54 -08:00
Erica Fischer 07ef9f4a3c Almost the right standard deviation 2023-02-09 12:04:44 -08:00
Erica Fischer 8264b4096a Combine mean and categoricality across reader threads 2023-02-09 10:39:30 -08:00
Erica Fischer fba86ebe24 Make struct coalesce a serial_feature wrapper rather than its own thing 2023-02-08 16:02:26 -08:00
Erica Fischer 4cd40c72ea Make struct partial a serial_feature wrapper rather than its own thing 2023-02-08 15:49:42 -08:00
Erica Fischer de607d1440 Preparing to sort by interestingness 2023-02-08 15:49:36 -08:00
Erica Fischer c6379d74d2 Merge branch 'order-feature-dropping' into categoricality 2023-02-08 14:48:38 -08:00
Erica Fischer 0c31280339 Merge branch 'main' into categoricality 2023-02-07 12:32:51 -08:00
Erica Fischer 71df08511e Add test 2023-02-01 21:18:59 -08:00
Erica Fischer efa209d175 Fill out more necessary fields of the partial feature 2023-02-01 21:16:23 -08:00
Erica Fischer 9f97397e21 Works, but the geometry is getting mangled 2023-02-01 20:59:30 -08:00
Erica Fischer 61b5475015 Duplicate some code so we can order features in partial form 2023-02-01 20:27:42 -08:00
Erica Fischer 3ae800745c Use circular mean to determine how categorical each attribute is 2022-12-07 11:05:10 -08:00
9 changed files with 1895 additions and 471 deletions
+9 -2
View File
@@ -2496,6 +2496,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
perror("close sorted index");
}
// provisional merged tilestats, so the stats for each attribute can be used during tiling
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
/* Traverse and split the geometries for each zoom level */
struct stat geomst;
@@ -2518,7 +2521,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
std::atomic<unsigned> midx(0);
std::atomic<unsigned> midy(0);
std::vector<strategy> strategies;
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies);
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, merged_lm, prefilter, postfilter, attribute_accum, filter, strategies);
if (maxzoom != written) {
if (written > minzoom) {
@@ -2573,7 +2576,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
midlon = maxlon;
}
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
// merge again, because new entries can have been added during tiling
merged_lm = merge_layermaps(layermaps);
for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) {
ai->second.minzoom = minzoom;
@@ -2855,6 +2859,7 @@ int main(int argc, char **argv) {
{"order-descending-by", required_argument, 0, '~'},
{"order-smallest-first", no_argument, 0, '~'},
{"order-largest-first", no_argument, 0, '~'},
{"order-interesting-first", no_argument, 0, '~'},
{"Adding calculated attributes", 0, 0, 0},
{"calculate-feature-density", no_argument, &additional[A_CALCULATE_FEATURE_DENSITY], 1},
@@ -2999,6 +3004,8 @@ int main(int argc, char **argv) {
} else if (strcmp(opt, "order-largest-first") == 0) {
order_by.push_back(order_field(ORDER_BY_SIZE, true));
order_by_size = true;
} else if (strcmp(opt, "order-interesting-first") == 0) {
order_by.push_back(order_field(ORDER_BY_INTERESTINGNESS, true));
} else if (strcmp(opt, "simplification-at-maximum-zoom") == 0) {
maxzoom_simplification = atof_require(optarg, "Mazoom simplification");
if (maxzoom_simplification <= 0) {
+2 -1
View File
@@ -69,8 +69,9 @@ struct order_field {
extern std::vector<order_field> order_by;
// not legal UTF-8, so can't appear as a real attribute name
#define ORDER_BY_SIZE "\200size"
extern bool order_by_size;
#define ORDER_BY_SIZE "\200size"
#define ORDER_BY_INTERESTINGNESS "\200interestingness"
int mkstemp_cloexec(char *name);
FILE *fopen_oflag(const char *name, const char *mode, int oflag);
+84
View File
@@ -285,6 +285,18 @@ void tilestats(std::map<std::string, layermap_entry> const &layermap1, size_t el
state.nospace = true;
state.json_write_string(attribute.first);
double xd = attribute.second.xsum / attribute.second.count;
double yd = attribute.second.ysum / attribute.second.count;
double d = sqrt(xd * xd + yd * yd);
#if 0
printf("%s %0.6f", attribute.first.c_str(), d);
if (attribute.second.numeric_count != 0) {
printf(" %f %f", attribute.second.mean, sqrt(attribute.second.m2 / attribute.second.numeric_count));
}
printf("\n");
#endif
size_t val_count = attribute.second.sample_values.size();
if (val_count > max_tilestats_sample_values) {
val_count = max_tilestats_sample_values;
@@ -840,6 +852,45 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
if (fk->second.max > fk2->second.max) {
fk2->second.max = fk->second.max;
}
fk2->second.xsum += fk->second.xsum;
fk2->second.ysum += fk->second.ysum;
fk2->second.count += fk->second.count;
if (fk->second.numeric_count + fk2->second.numeric_count > 0) {
// https://math.stackexchange.com/questions/2971315/how-do-i-combine-standard-deviations-of-two-groups
size_t n1 = fk->second.numeric_count;
size_t n2 = fk2->second.numeric_count;
if (n1 == 0) {
// fk2 is already correct
} else if (n2 == 0) {
// move fk to fk2
fk2->second.m2 = fk->second.m2;
} else {
double var_x1 = fk->second.m2 / n1;
double var_x2 = fk2->second.m2 / n2;
double mean_x1 = fk->second.mean;
double mean_x2 = fk2->second.mean;
double mean_x = (n1 * mean_x1 + n2 * mean_x2) / (n1 + n2);
// stack overflow uses n1 - 1 and n2 - 1
double q1 = (n1) * var_x1 + n1 * mean_x1 * mean_x1;
double q2 = (n2) * var_x2 + n2 * mean_x2 * mean_x2;
double qc = q1 + q2;
// stack overflow uses n1 + n2 - 1
double sc = sqrt((qc - (n1 + n2) * mean_x * mean_x) / (n1 + n2));
fk2->second.m2 = sc * sc * (n1 + n2);
}
fk2->second.mean = (
fk2->second.mean * fk2->second.numeric_count +
fk->second.mean * fk->second.numeric_count
) / (fk2->second.numeric_count + fk->second.numeric_count);
fk2->second.numeric_count += fk->second.numeric_count;
}
}
}
@@ -859,6 +910,20 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
return out;
}
double categorical_hash(std::string const &val) {
unsigned hash = 0;
for (size_t i = 0; i < val.size(); i++) {
// https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing
hash = hash * 2654435769 + val[i];
}
// extra multiply so even single-byte values are distributed
// around the circle rather than clumped together on one side.
hash = hash * 2654435769;
double angle = ((double) hash) / UINT_MAX * 2 * M_PI;
return angle;
}
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &attrib, type_and_string const &val) {
if (val.type == mvt_null) {
return;
@@ -895,5 +960,24 @@ void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, s
}
}
double angle = categorical_hash(val.string);
double x = cos(angle);
double y = sin(angle);
fka->second.xsum += x;
fka->second.ysum += y;
fka->second.count++;
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
if (val.type == mvt_double) {
double newValue = atof(val.string.c_str());
fka->second.numeric_count++;
double delta = newValue - fka->second.mean;
fka->second.mean += delta / fka->second.numeric_count;
double delta2 = newValue - fka->second.mean;
fka->second.m2 += delta * delta2;
}
fka->second.type |= (1 << val.type);
}
+11
View File
@@ -22,6 +22,16 @@ struct type_and_string_stats {
double min = INFINITY;
double max = -INFINITY;
int type = 0;
// for circular mean of value hashes
double xsum = 0;
double ysum = 0;
size_t count = 0;
// for mean and standard deviation
double mean = 0;
double numeric_count = 0;
double m2 = 0;
};
struct layermap_entry {
@@ -83,5 +93,6 @@ std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps, bool trunc);
void add_to_file_keys(std::map<std::string, type_and_string_stats> &file_keys, std::string const &layername, type_and_string const &val);
double categorical_hash(std::string const &value);
#endif
+3
View File
@@ -17,6 +17,7 @@
#include "options.hpp"
#include "main.hpp"
#include "pool.hpp"
#include "memfile.hpp"
#include "projection.hpp"
#include "evaluator.hpp"
#include "milo/dtoa_milo.h"
@@ -740,6 +741,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
}
}
sf.stringpool = r->poolfile->map;
long long geomstart = r->geompos;
sf.geometry = scaled_geometry;
serialize_feature(r->geomfile, &sf, &r->geompos, sst->fname, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y)), false);
+1
View File
@@ -70,6 +70,7 @@ struct serial_feature {
std::vector<serial_val> full_values{};
std::string layername = "";
bool dropped = false;
const char *stringpool; // the pool for this segment, where its keys and values are stored
};
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
+506 -467
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -63,7 +63,7 @@ struct strategy {
long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, std::atomic<strategy> *strategy);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const std::map<std::string, layermap_entry> &merged_layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);