Merge branch 'more-warnings' into rate-clarify

This commit is contained in:
Eric Fischer
2017-11-17 11:21:01 -08:00
24 changed files with 481 additions and 445 deletions
+154 -153
View File
@@ -76,21 +76,21 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
struct coalesce {
char *stringpool;
std::vector<long long> keys;
std::vector<long long> values;
std::vector<std::string> full_keys;
std::vector<serial_val> full_values;
drawvec geom;
unsigned long long index;
unsigned long long index2;
long long original_seq;
int type;
int m;
bool coalesced;
double spacing;
bool has_id;
unsigned long long id;
char *stringpool = NULL;
std::vector<long long> keys = std::vector<long long>();
std::vector<long long> values = std::vector<long long>();
std::vector<std::string> full_keys = std::vector<std::string>();
std::vector<serial_val> full_values = std::vector<serial_val>();
drawvec geom = drawvec();
unsigned long long index = 0;
unsigned long long index2 = 0;
long long original_seq = 0;
int type = 0;
int m = 0;
bool coalesced = false;
double spacing = 0;
bool has_id = false;
unsigned long long id = 0;
bool operator<(const coalesce &o) const {
int cmp = coalindexcmp(this, &o);
@@ -200,7 +200,7 @@ int metacmp(int m1, const std::vector<long long> &keys1, const std::vector<long
}
}
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, long long extent) {
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
int xo, yo;
int span = 1 << (nextzoom - z);
@@ -309,34 +309,34 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u
}
struct partial {
std::vector<drawvec> geoms;
std::vector<long long> keys;
std::vector<long long> values;
std::vector<std::string> full_keys;
std::vector<serial_val> full_values;
std::vector<ssize_t> arc_polygon;
long long layer;
long long original_seq;
unsigned long long index;
unsigned long long index2;
int m;
int segment;
bool reduced;
int z;
int line_detail;
int maxzoom;
double spacing;
double simplification;
signed char t;
unsigned long long id;
bool has_id;
ssize_t renamed;
std::vector<drawvec> geoms = std::vector<drawvec>();
std::vector<long long> keys = std::vector<long long>();
std::vector<long long> values = std::vector<long long>();
std::vector<std::string> full_keys = std::vector<std::string>();
std::vector<serial_val> full_values = std::vector<serial_val>();
std::vector<ssize_t> arc_polygon = std::vector<ssize_t>();
long long layer = 0;
long long original_seq = 0;
unsigned long long index = 0;
unsigned long long index2 = 0;
int m = 0;
int segment = 0;
bool reduced = 0;
int z = 0;
int line_detail = 0;
int maxzoom = 0;
double spacing = 0;
double simplification = 0;
signed char t = 0;
unsigned long long id = 0;
bool has_id = 0;
ssize_t renamed = 0;
};
struct partial_arg {
std::vector<struct partial> *partials;
int task;
int tasks;
std::vector<struct partial> *partials = NULL;
int task = 0;
int tasks = 0;
};
drawvec revive_polygon(drawvec &geom, double area, int z, int detail) {
@@ -442,9 +442,9 @@ void *partial_feature_worker(void *v) {
// Give Clipper a chance to try to fix it.
for (size_t g = 0; g < geoms.size(); g++) {
drawvec before = geoms[g];
geoms[g] = clean_or_clip_poly(geoms[g], 0, 0, 0, false);
geoms[g] = clean_or_clip_poly(geoms[g], 0, 0, false);
if (additional[A_DEBUG_POLYGON]) {
check_polygon(geoms[g], before);
check_polygon(geoms[g]);
}
if (geoms[g].size() < 3) {
@@ -523,11 +523,11 @@ static drawvec reverse_subring(drawvec const &dv) {
}
struct edge {
unsigned x1;
unsigned y1;
unsigned x2;
unsigned y2;
unsigned ring;
unsigned x1 = 0;
unsigned y1 = 0;
unsigned x2 = 0;
unsigned y2 = 0;
unsigned ring = 0;
edge(unsigned _x1, unsigned _y1, unsigned _x2, unsigned _y2, unsigned _ring) {
x1 = _x1;
@@ -872,10 +872,10 @@ bool find_common_edges(std::vector<partial> &partials, int z, int line_detail, d
// If necessary, merge some adjacent polygons into some other polygons
struct merge_order {
ssize_t edge;
unsigned long long gap;
size_t p1;
size_t p2;
ssize_t edge = 0;
unsigned long long gap = 0;
size_t p1 = 0;
size_t p2 = 0;
bool operator<(const merge_order &m) const {
return gap < m.gap;
@@ -1127,53 +1127,51 @@ long long choose_minextent(std::vector<long long> &extents, double f) {
}
struct write_tile_args {
struct task *tasks;
char *metabase;
char *stringpool;
int min_detail;
int basezoom;
sqlite3 *outdb;
const char *outdir;
double droprate;
int buffer;
const char *fname;
FILE **geomfile;
double todo;
volatile long long *along;
double gamma;
double gamma_out;
int child_shards;
int *geomfd;
off_t *geom_size;
volatile unsigned *midx;
volatile unsigned *midy;
int maxzoom;
int minzoom;
int full_detail;
int low_detail;
double simplification;
volatile long long *most;
long long *meta_off;
long long *pool_off;
unsigned *initial_x;
unsigned *initial_y;
volatile int *running;
int err;
std::vector<std::map<std::string, layermap_entry>> *layermaps;
std::vector<std::vector<std::string>> *layer_unmaps;
size_t pass;
size_t passes;
unsigned long long mingap;
unsigned long long mingap_out;
long long minextent;
long long minextent_out;
double fraction;
double fraction_out;
const char *prefilter;
const char *postfilter;
bool still_dropping;
int wrote_zoom;
size_t tiling_seg;
struct task *tasks = NULL;
char *metabase = NULL;
char *stringpool = NULL;
int min_detail = 0;
sqlite3 *outdb = NULL;
const char *outdir = NULL;
int buffer = 0;
const char *fname = NULL;
FILE **geomfile = NULL;
double todo = 0;
volatile long long *along = NULL;
double gamma = 0;
double gamma_out = 0;
int child_shards = 0;
int *geomfd = NULL;
off_t *geom_size = NULL;
volatile unsigned *midx = NULL;
volatile unsigned *midy = NULL;
int maxzoom = 0;
int minzoom = 0;
int full_detail = 0;
int low_detail = 0;
double simplification = 0;
volatile long long *most = NULL;
long long *meta_off = NULL;
long long *pool_off = NULL;
unsigned *initial_x = NULL;
unsigned *initial_y = NULL;
volatile int *running = NULL;
int err = 0;
std::vector<std::map<std::string, layermap_entry>> *layermaps = NULL;
std::vector<std::vector<std::string>> *layer_unmaps = NULL;
size_t pass = 0;
size_t passes = 0;
unsigned long long mingap = 0;
unsigned long long mingap_out = 0;
long long minextent = 0;
long long minextent_out = 0;
double fraction = 0;
double fraction_out = 0;
const char *prefilter = NULL;
const char *postfilter = NULL;
bool still_dropping = false;
int wrote_zoom = 0;
size_t tiling_seg = 0;
};
bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
@@ -1233,7 +1231,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
// that are duplicated across the date line
if (prevent[P_DUPLICATION] && z != 0) {
if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z, buffer)) {
if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z)) {
// sf.geometry is unchanged
} else {
sf.geometry.clear();
@@ -1252,7 +1250,7 @@ bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
return false;
}
serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, int line_detail, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) {
serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, long long *meta_off, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, size_t passes, volatile long long *along, long long alongminus, int buffer, int *within, bool *first_time, FILE **geomfile, long long *geompos, volatile double *oprogress, double todo, const char *fname, int child_shards) {
while (1) {
serial_feature sf = deserialize_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y);
if (sf.t < 0) {
@@ -1279,7 +1277,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
if (*first_time && pass == 1) { /* only write out the next zoom once, even if we retry */
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.metapos, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.m, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.extent);
}
}
@@ -1302,46 +1300,45 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase,
}
struct run_prefilter_args {
FILE *geoms;
long long *geompos_in;
char *metabase;
long long *meta_off;
int z;
unsigned tx;
unsigned ty;
unsigned *initial_x;
unsigned *initial_y;
long long *original_features;
long long *unclipped_features;
int nextzoom;
int maxzoom;
int minzoom;
int max_zoom_increment;
size_t pass;
size_t passes;
volatile long long *along;
long long alongminus;
int buffer;
int *within;
bool *first_time;
int line_detail;
FILE **geomfile;
long long *geompos;
volatile double *oprogress;
double todo;
const char *fname;
int child_shards;
std::vector<std::vector<std::string>> *layer_unmaps;
char *stringpool;
long long *pool_off;
FILE *prefilter_fp;
FILE *geoms = NULL;
long long *geompos_in = NULL;
char *metabase = NULL;
long long *meta_off = NULL;
int z = 0;
unsigned tx = 0;
unsigned ty = 0;
unsigned *initial_x = 0;
unsigned *initial_y = 0;
long long *original_features = 0;
long long *unclipped_features = 0;
int nextzoom = 0;
int maxzoom = 0;
int minzoom = 0;
int max_zoom_increment = 0;
size_t pass = 0;
size_t passes = 0;
volatile long long *along = 0;
long long alongminus = 0;
int buffer = 0;
int *within = NULL;
bool *first_time = NULL;
FILE **geomfile = NULL;
long long *geompos = NULL;
volatile double *oprogress = NULL;
double todo = 0;
const char *fname = 0;
int child_shards = 0;
std::vector<std::vector<std::string>> *layer_unmaps = NULL;
char *stringpool = NULL;
long long *pool_off = NULL;
FILE *prefilter_fp = NULL;
};
void *run_prefilter(void *v) {
run_prefilter_args *rpa = (run_prefilter_args *) v;
while (1) {
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->line_detail, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards);
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->metabase, rpa->meta_off, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->passes, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->first_time, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards);
if (sf.t < 0) {
break;
}
@@ -1392,7 +1389,7 @@ void *run_prefilter(void *v) {
return NULL;
}
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) {
long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, write_tile_args *arg) {
int line_detail;
double merge_fraction = 1;
double mingap_fraction = 1;
@@ -1497,7 +1494,6 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
rpa.buffer = buffer;
rpa.within = within;
rpa.first_time = &first_time;
rpa.line_detail = line_detail;
rpa.geomfile = geomfile;
rpa.geompos = geompos;
rpa.oprogress = &oprogress;
@@ -1526,7 +1522,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
serial_feature sf;
if (prefilter == NULL) {
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, line_detail, geomfile, geompos, &oprogress, todo, fname, child_shards);
sf = next_feature(geoms, geompos_in, metabase, meta_off, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, passes, along, alongminus, buffer, within, &first_time, geomfile, geompos, &oprogress, todo, fname, child_shards);
} else {
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
}
@@ -1684,7 +1680,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
}
pthread_t pthreads[tasks];
partial_arg args[tasks];
std::vector<partial_arg> args;
args.resize(tasks);
for (int i = 0; i < tasks; i++) {
args[i].task = i;
args[i].tasks = tasks;
@@ -1809,7 +1806,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
if (layer_features[x].type == VT_POLYGON) {
if (layer_features[x].coalesced) {
layer_features[x].geom = clean_or_clip_poly(layer_features[x].geom, 0, 0, 0, false);
layer_features[x].geom = clean_or_clip_poly(layer_features[x].geom, 0, 0, false);
}
layer_features[x].geom = close_poly(layer_features[x].geom);
@@ -2094,8 +2091,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s
}
struct task {
int fileno;
struct task *next;
int fileno = 0;
struct task *next = NULL;
};
void *run_thread(void *vargs) {
@@ -2138,7 +2135,7 @@ void *run_thread(void *vargs) {
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->outdb, arg->outdir, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg);
long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->passes, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg);
if (len < 0) {
int *err = &arg->err;
@@ -2203,7 +2200,7 @@ void *run_thread(void *vargs) {
return NULL;
}
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, int basezoom, sqlite3 *outdb, const char *outdir, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter) {
int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter) {
// The existing layermaps are one table per input thread.
// We need to add another one per *tiling* thread so that it can be
// safely changed during tiling.
@@ -2281,12 +2278,17 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
// Assign temporary files to threads
struct task tasks[TEMP_FILES];
std::vector<struct task> tasks;
tasks.resize(TEMP_FILES);
struct dispatch {
struct task *tasks;
long long todo;
struct dispatch *next;
} dispatches[threads];
struct task *tasks = NULL;
long long todo = 0;
struct dispatch *next = NULL;
};
std::vector<struct dispatch> dispatches;
dispatches.resize(threads);
struct dispatch *dispatch_head = &dispatches[0];
for (size_t j = 0; j < threads; j++) {
dispatches[j].tasks = NULL;
@@ -2336,7 +2338,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
for (size_t pass = start; pass < 2; pass++) {
pthread_t pthreads[threads];
write_tile_args args[threads];
std::vector<write_tile_args> args;
args.resize(threads);
int running = threads;
long long along = 0;
@@ -2344,10 +2347,8 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
args[thread].metabase = metabase;
args[thread].stringpool = stringpool;
args[thread].min_detail = min_detail;
args[thread].basezoom = basezoom;
args[thread].outdb = outdb; // locked with db_lock
args[thread].outdir = outdir;
args[thread].droprate = droprate;
args[thread].buffer = buffer;
args[thread].fname = fname;
args[thread].geomfile = sub + thread * (TEMP_FILES / threads);