Another option for plain fractional dropping, but across the whole zoom

This commit is contained in:
Eric Fischer
2016-11-04 12:26:13 -07:00
parent ee48be26e0
commit 38ce49d2d4
7 changed files with 15078 additions and 10 deletions
+19 -6
View File
@@ -1209,11 +1209,12 @@ struct write_tile_args {
size_t passes;
unsigned long long mingap;
unsigned long long mingap_out;
double fraction;
double fraction_out;
};
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, write_tile_args *arg) {
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t pass, size_t passes, unsigned long long mingap, double fraction, write_tile_args *arg) {
int line_detail;
double fraction = 1;
double merge_fraction = 1;
double mingap_fraction = 1;
@@ -1775,11 +1776,14 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
}
line_detail++;
continue;
} else if (prevent[P_DYNAMIC_DROP]) {
} else if (prevent[P_DYNAMIC_DROP] || additional[A_DROP_FRACTION_AS_NEEDED]) {
fraction = fraction * 200000 / totalsize * 0.95;
if (!quiet) {
fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
}
if (additional[A_DROP_FRACTION_AS_NEEDED] && fraction < arg->fraction_out) {
arg->fraction_out = fraction;
}
line_detail++; // to keep it the same when the loop decrements it
continue;
} else {
@@ -1826,7 +1830,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
fprintf(stderr, "Going to try keeping the sparsest %0.2f%% of the features to make it fit\n", mingap_fraction * 100.0);
}
line_detail++;
} else if (prevent[P_DYNAMIC_DROP]) {
} else if (prevent[P_DYNAMIC_DROP] || additional[A_DROP_FRACTION_AS_NEEDED]) {
// The 95% is a guess to avoid too many retries
// and probably actually varies based on how much duplicated metadata there is
@@ -1834,6 +1838,9 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
if (!quiet) {
fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
}
if (additional[A_DROP_FRACTION_AS_NEEDED] && fraction < arg->fraction_out) {
arg->fraction_out = fraction;
}
line_detail++; // to keep it the same when the loop decrements it
}
} else {
@@ -1905,7 +1912,7 @@ void *run_thread(void *vargs) {
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->pass, arg->passes, arg->mingap, arg->fraction, arg);
if (len < 0) {
int *err = &arg->err;
@@ -2084,12 +2091,13 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
int err = INT_MAX;
size_t start = 1;
if (additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_INCREASE_SPACING_AS_NEEDED]) {
if (additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_INCREASE_SPACING_AS_NEEDED] || additional[A_DROP_FRACTION_AS_NEEDED]) {
start = 0;
}
double zoom_gamma = gamma;
unsigned long long zoom_mingap = 0;
double zoom_fraction = 1;
for (size_t pass = start; pass < 2; pass++) {
pthread_t pthreads[threads];
@@ -2113,6 +2121,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
args[thread].gamma_out = zoom_gamma;
args[thread].mingap = zoom_mingap;
args[thread].mingap_out = zoom_mingap;
args[thread].fraction = zoom_fraction;
args[thread].fraction_out = zoom_fraction;
args[thread].child_shards = TEMP_FILES / threads;
args[thread].simplification = simplification;
@@ -2160,6 +2170,9 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
if (args[thread].mingap_out > zoom_mingap) {
zoom_mingap = args[thread].mingap_out;
}
if (args[thread].fraction_out < zoom_fraction) {
zoom_fraction = args[thread].fraction_out;
}
}
}