mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Erica Fischer
d05f8f8e4c
This was an extra level of attribute indirection (features point to metadata records which point to key and value strings) which was intended to reduce the size of temporary storage for features with large numbers of attributes that were also spread across large numbers of tiles at maxzoom. For other kinds of features, the extra indirection slowed things down instead, and, especially when maxzoom guessing was being used, many more features were having their metadata externalized than could actually benefit from it.
3209 lines
94 KiB
C++
3209 lines
94 KiB
C++
#ifdef __APPLE__
|
|
#define _DARWIN_UNLIMITED_STREAMS
|
|
#endif
|
|
|
|
#include <iostream>
|
|
#include <fstream>
|
|
#include <string>
|
|
#include <stack>
|
|
#include <vector>
|
|
#include <map>
|
|
#include <set>
|
|
#include <algorithm>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
#include <limits.h>
|
|
#include <zlib.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/types.h>
|
|
#include <sys/mman.h>
|
|
#include <cmath>
|
|
#include <sqlite3.h>
|
|
#include <pthread.h>
|
|
#include <errno.h>
|
|
#include <time.h>
|
|
#include <fcntl.h>
|
|
#include <sys/wait.h>
|
|
#include "mvt.hpp"
|
|
#include "mbtiles.hpp"
|
|
#include "dirtiles.hpp"
|
|
#include "geometry.hpp"
|
|
#include "tile.hpp"
|
|
#include "pool.hpp"
|
|
#include "projection.hpp"
|
|
#include "serial.hpp"
|
|
#include "options.hpp"
|
|
#include "main.hpp"
|
|
#include "write_json.hpp"
|
|
#include "milo/dtoa_milo.h"
|
|
#include "evaluator.hpp"
|
|
#include "errors.hpp"
|
|
|
|
extern "C" {
|
|
#include "jsonpull/jsonpull.h"
|
|
}
|
|
|
|
#include "plugin.hpp"
|
|
|
|
#define CMD_BITS 3
|
|
|
|
// Offset coordinates to keep them positive
|
|
#define COORD_OFFSET (4LL << 32)
|
|
#define SHIFT_RIGHT(a) ((((a) + COORD_OFFSET) >> geometry_scale) - (COORD_OFFSET >> geometry_scale))
|
|
|
|
#define XSTRINGIFY(s) STRINGIFY(s)
|
|
#define STRINGIFY(s) #s
|
|
|
|
pthread_mutex_t db_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
pthread_mutex_t var_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
std::vector<mvt_geometry> to_feature(drawvec &geom) {
|
|
std::vector<mvt_geometry> out;
|
|
|
|
for (size_t i = 0; i < geom.size(); i++) {
|
|
out.push_back(mvt_geometry(geom[i].op, geom[i].x, geom[i].y));
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
bool draws_something(drawvec &geom) {
|
|
for (size_t i = 1; i < geom.size(); i++) {
|
|
if (geom[i].op == VT_LINETO && (geom[i].x != geom[i - 1].x || geom[i].y != geom[i - 1].y)) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static int metacmp(const std::vector<long long> &keys1, const std::vector<long long> &values1, char *stringpool1, const std::vector<long long> &keys2, const std::vector<long long> &values2, char *stringpool2);
|
|
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2);
|
|
|
|
struct coalesce {
|
|
char *stringpool = NULL;
|
|
std::vector<long long> keys = std::vector<long long>();
|
|
std::vector<long long> values = std::vector<long long>();
|
|
std::vector<std::string> full_keys = std::vector<std::string>();
|
|
std::vector<serial_val> full_values = std::vector<serial_val>();
|
|
drawvec geom = drawvec();
|
|
unsigned long long index = 0;
|
|
long long original_seq = 0;
|
|
int type = 0;
|
|
bool coalesced = false;
|
|
double spacing = 0;
|
|
bool has_id = false;
|
|
unsigned long long id = 0;
|
|
long long extent = 0;
|
|
|
|
bool operator<(const coalesce &o) const {
|
|
int cmp = coalindexcmp(this, &o);
|
|
if (cmp < 0) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
};
|
|
|
|
struct preservecmp {
|
|
bool operator()(const struct coalesce &a, const struct coalesce &b) {
|
|
return a.original_seq < b.original_seq;
|
|
}
|
|
} preservecmp;
|
|
|
|
int coalcmp(const void *v1, const void *v2) {
|
|
const struct coalesce *c1 = (const struct coalesce *) v1;
|
|
const struct coalesce *c2 = (const struct coalesce *) v2;
|
|
|
|
int cmp = c1->type - c2->type;
|
|
if (cmp != 0) {
|
|
return cmp;
|
|
}
|
|
|
|
if (c1->has_id != c2->has_id) {
|
|
return (int) c1->has_id - (int) c2->has_id;
|
|
}
|
|
|
|
if (c1->has_id && c2->has_id) {
|
|
if (c1->id < c2->id) {
|
|
return -1;
|
|
}
|
|
if (c1->id > c2->id) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
cmp = metacmp(c1->keys, c1->values, c1->stringpool, c2->keys, c2->values, c2->stringpool);
|
|
if (cmp != 0) {
|
|
return cmp;
|
|
}
|
|
|
|
if (c1->full_keys.size() < c2->full_keys.size()) {
|
|
return -1;
|
|
} else if (c1->full_keys.size() > c2->full_keys.size()) {
|
|
return 1;
|
|
}
|
|
|
|
for (size_t i = 0; i < c1->full_keys.size(); i++) {
|
|
if (c1->full_keys[i] < c2->full_keys[i]) {
|
|
return -1;
|
|
} else if (c1->full_keys[i] > c2->full_keys[i]) {
|
|
return 1;
|
|
}
|
|
|
|
if (c1->full_values[i].type < c2->full_values[i].type) {
|
|
return -1;
|
|
} else if (c1->full_values[i].type > c2->full_values[i].type) {
|
|
return 1;
|
|
}
|
|
|
|
if (c1->full_values[i].s < c2->full_values[i].s) {
|
|
return -1;
|
|
} else if (c1->full_values[i].s > c2->full_values[i].s) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) {
|
|
int cmp = coalcmp((const void *) c1, (const void *) c2);
|
|
|
|
if (cmp == 0) {
|
|
if (c1->index < c2->index) {
|
|
return -1;
|
|
} else if (c1->index > c2->index) {
|
|
return 1;
|
|
}
|
|
|
|
if (c1->geom < c2->geom) {
|
|
return -1;
|
|
} else if (c1->geom > c2->geom) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
return cmp;
|
|
}
|
|
|
|
mvt_value retrieve_string(long long off, const char *stringpool, int *otype) {
|
|
int type = stringpool[off];
|
|
const char *s = stringpool + off + 1;
|
|
|
|
if (otype != NULL) {
|
|
*otype = type;
|
|
}
|
|
|
|
return stringified_to_mvt_value(type, s);
|
|
}
|
|
|
|
void decode_meta(std::vector<long long> const &metakeys, std::vector<long long> const &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) {
|
|
size_t i;
|
|
for (i = 0; i < metakeys.size(); i++) {
|
|
int otype;
|
|
mvt_value key = retrieve_string(metakeys[i], stringpool, NULL);
|
|
mvt_value value = retrieve_string(metavals[i], stringpool, &otype);
|
|
|
|
layer.tag(feature, key.string_value, value);
|
|
}
|
|
}
|
|
|
|
static int metacmp(const std::vector<long long> &keys1, const std::vector<long long> &values1, char *stringpool1, const std::vector<long long> &keys2, const std::vector<long long> &values2, char *stringpool2) {
|
|
size_t i;
|
|
for (i = 0; i < keys1.size() && i < keys2.size(); i++) {
|
|
mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL);
|
|
mvt_value key2 = retrieve_string(keys2[i], stringpool2, NULL);
|
|
|
|
if (key1.string_value < key2.string_value) {
|
|
return -1;
|
|
} else if (key1.string_value > key2.string_value) {
|
|
return 1;
|
|
}
|
|
|
|
long long off1 = values1[i];
|
|
int type1 = stringpool1[off1];
|
|
char *s1 = stringpool1 + off1 + 1;
|
|
|
|
long long off2 = values2[i];
|
|
int type2 = stringpool2[off2];
|
|
char *s2 = stringpool2 + off2 + 1;
|
|
|
|
if (type1 != type2) {
|
|
return type1 - type2;
|
|
}
|
|
int cmp = strcmp(s1, s2);
|
|
if (cmp != 0) {
|
|
return cmp;
|
|
}
|
|
}
|
|
|
|
if (keys1.size() < keys2.size()) {
|
|
return -1;
|
|
} else if (keys1.size() > keys2.size()) {
|
|
return 1;
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
static mvt_value find_attribute_value(const struct coalesce *c1, std::string key) {
|
|
if (key == ORDER_BY_SIZE) {
|
|
mvt_value v;
|
|
v.type = mvt_double;
|
|
v.numeric_value.double_value = c1->extent;
|
|
return v;
|
|
}
|
|
|
|
const std::vector<long long> &keys1 = c1->keys;
|
|
const std::vector<long long> &values1 = c1->values;
|
|
const char *stringpool1 = c1->stringpool;
|
|
|
|
for (size_t i = 0; i < keys1.size(); i++) {
|
|
mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL);
|
|
if (key == key1.string_value) {
|
|
return retrieve_string(values1[i], stringpool1, NULL);
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < c1->full_keys.size(); i++) {
|
|
if (c1->full_keys[i] == key) {
|
|
return stringified_to_mvt_value(c1->full_values[i].type, c1->full_values[i].s.c_str());
|
|
}
|
|
}
|
|
|
|
mvt_value v;
|
|
v.type = mvt_null;
|
|
v.numeric_value.null_value = 0;
|
|
return v;
|
|
}
|
|
|
|
static mvt_value coerce_double(mvt_value v) {
|
|
if (v.type == mvt_int) {
|
|
v.type = mvt_double;
|
|
v.numeric_value.double_value = v.numeric_value.int_value;
|
|
} else if (v.type == mvt_uint) {
|
|
v.type = mvt_double;
|
|
v.numeric_value.double_value = v.numeric_value.uint_value;
|
|
} else if (v.type == mvt_sint) {
|
|
v.type = mvt_double;
|
|
v.numeric_value.double_value = v.numeric_value.sint_value;
|
|
} else if (v.type == mvt_float) {
|
|
v.type = mvt_double;
|
|
v.numeric_value.double_value = v.numeric_value.float_value;
|
|
}
|
|
|
|
return v;
|
|
}
|
|
|
|
struct ordercmp {
|
|
bool operator()(const struct coalesce &a, const struct coalesce &b) {
|
|
for (size_t i = 0; i < order_by.size(); i++) {
|
|
mvt_value v1 = coerce_double(find_attribute_value(&a, order_by[i].name));
|
|
mvt_value v2 = coerce_double(find_attribute_value(&b, order_by[i].name));
|
|
|
|
if (order_by[i].descending) {
|
|
if (v2 < v1) {
|
|
return true;
|
|
} else if (v1 < v2) {
|
|
return false;
|
|
} // else they are equal, so continue to the next attribute
|
|
} else {
|
|
if (v1 < v2) {
|
|
return true;
|
|
} else if (v2 < v1) {
|
|
return false;
|
|
} // else they are equal, so continue to the next attribute
|
|
}
|
|
}
|
|
|
|
return false; // greater than or equal
|
|
}
|
|
} ordercmp;
|
|
|
|
void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int *within, std::atomic<long long> *geompos, FILE **geomfile, const char *fname, signed char t, int layer, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, std::vector<long long> &metakeys, std::vector<long long> &metavals, bool has_id, unsigned long long id, unsigned long long index, unsigned long long label_point, long long extent) {
|
|
if (geom.size() > 0 && (nextzoom <= maxzoom || additional[A_EXTEND_ZOOMS])) {
|
|
int xo, yo;
|
|
int span = 1 << (nextzoom - z);
|
|
|
|
// Get the feature bounding box in pixel (256) coordinates at the child zoom
|
|
// in order to calculate which sub-tiles it can touch including the buffer.
|
|
long long bbox2[4];
|
|
int k;
|
|
for (k = 0; k < 4; k++) {
|
|
// Division instead of right-shift because coordinates can be negative
|
|
bbox2[k] = bbox[k] / (1 << (32 - nextzoom - 8));
|
|
}
|
|
// Decrement the top and left edges so that any features that are
|
|
// touching the edge can potentially be included in the adjacent tiles too.
|
|
bbox2[0] -= buffer + 1;
|
|
bbox2[1] -= buffer + 1;
|
|
bbox2[2] += buffer;
|
|
bbox2[3] += buffer;
|
|
|
|
for (k = 0; k < 4; k++) {
|
|
if (bbox2[k] < 0) {
|
|
bbox2[k] = 0;
|
|
}
|
|
if (bbox2[k] >= 256 * span) {
|
|
bbox2[k] = 256 * (span - 1);
|
|
}
|
|
|
|
bbox2[k] /= 256;
|
|
}
|
|
|
|
// Offset from tile coordinates back to world coordinates
|
|
unsigned sx = 0, sy = 0;
|
|
if (z != 0) {
|
|
sx = tx << (32 - z);
|
|
sy = ty << (32 - z);
|
|
}
|
|
|
|
drawvec geom2;
|
|
for (size_t i = 0; i < geom.size(); i++) {
|
|
geom2.push_back(draw(geom[i].op, SHIFT_RIGHT(geom[i].x + sx), SHIFT_RIGHT(geom[i].y + sy)));
|
|
}
|
|
|
|
for (xo = bbox2[0]; xo <= bbox2[2]; xo++) {
|
|
for (yo = bbox2[1]; yo <= bbox2[3]; yo++) {
|
|
unsigned jx = tx * span + xo;
|
|
unsigned jy = ty * span + yo;
|
|
|
|
// j is the shard that the child tile's data is being written to.
|
|
//
|
|
// Be careful: We can't jump more zoom levels than max_zoom_increment
|
|
// because that could break the constraint that each of the children
|
|
// of the current tile must have its own shard, because the data for
|
|
// the child tile must be contiguous within the shard.
|
|
//
|
|
// But it's OK to spread children across all the shards, not just
|
|
// the four that would normally result from splitting one tile,
|
|
// because it will go through all the shards when it does the
|
|
// next zoom.
|
|
//
|
|
// If child_shards is a power of 2 but not a power of 4, this will
|
|
// shard X more widely than Y. XXX Is there a better way to do this
|
|
// without causing collisions?
|
|
|
|
int j = ((jx << max_zoom_increment) |
|
|
((jy & ((1 << max_zoom_increment) - 1)))) &
|
|
(child_shards - 1);
|
|
|
|
{
|
|
if (!within[j]) {
|
|
serialize_int(geomfile[j], nextzoom, &geompos[j], fname);
|
|
serialize_uint(geomfile[j], tx * span + xo, &geompos[j], fname);
|
|
serialize_uint(geomfile[j], ty * span + yo, &geompos[j], fname);
|
|
within[j] = 1;
|
|
}
|
|
|
|
serial_feature sf;
|
|
sf.layer = layer;
|
|
sf.segment = segment;
|
|
sf.seq = seq;
|
|
sf.t = t;
|
|
sf.has_id = has_id;
|
|
sf.id = id;
|
|
sf.has_tippecanoe_minzoom = tippecanoe_minzoom != -1;
|
|
sf.tippecanoe_minzoom = tippecanoe_minzoom;
|
|
sf.has_tippecanoe_maxzoom = tippecanoe_maxzoom != -1;
|
|
sf.tippecanoe_maxzoom = tippecanoe_maxzoom;
|
|
sf.geometry = geom2;
|
|
sf.index = index;
|
|
sf.label_point = label_point;
|
|
sf.extent = extent;
|
|
sf.feature_minzoom = feature_minzoom;
|
|
|
|
for (size_t i = 0; i < metakeys.size(); i++) {
|
|
sf.keys.push_back(metakeys[i]);
|
|
sf.values.push_back(metavals[i]);
|
|
}
|
|
|
|
serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
struct accum_state {
|
|
double sum = 0;
|
|
double count = 0;
|
|
};
|
|
|
|
struct partial {
|
|
std::vector<drawvec> geoms = std::vector<drawvec>();
|
|
std::vector<long long> keys = std::vector<long long>();
|
|
std::vector<long long> values = std::vector<long long>();
|
|
std::vector<std::string> full_keys = std::vector<std::string>();
|
|
std::vector<serial_val> full_values = std::vector<serial_val>();
|
|
std::vector<ssize_t> arc_polygon = std::vector<ssize_t>();
|
|
long long layer = 0;
|
|
long long original_seq = 0;
|
|
unsigned long long index = 0;
|
|
unsigned long long label_point = 0;
|
|
int segment = 0;
|
|
bool reduced = 0;
|
|
bool coalesced = 0;
|
|
int z = 0;
|
|
int tx = 0;
|
|
int ty = 0;
|
|
int line_detail = 0;
|
|
int extra_detail = 0;
|
|
int maxzoom = 0;
|
|
double spacing = 0;
|
|
double simplification = 0;
|
|
signed char t = 0;
|
|
unsigned long long id = 0;
|
|
bool has_id = 0;
|
|
ssize_t renamed = 0;
|
|
long long extent = 0;
|
|
long long clustered = 0;
|
|
std::set<std::string> need_tilestats;
|
|
std::map<std::string, accum_state> attribute_accum_state;
|
|
};
|
|
|
|
struct partial_arg {
|
|
std::vector<struct partial> *partials = NULL;
|
|
int task = 0;
|
|
int tasks = 0;
|
|
drawvec *shared_nodes;
|
|
};
|
|
|
|
drawvec revive_polygon(drawvec &geom, double area, int z, int detail) {
|
|
// From area in world coordinates to area in tile coordinates
|
|
long long divisor = 1LL << (32 - detail - z);
|
|
area /= divisor * divisor;
|
|
|
|
if (area == 0) {
|
|
return drawvec();
|
|
}
|
|
|
|
int height = ceil(sqrt(area));
|
|
int width = round(area / height);
|
|
if (width == 0) {
|
|
width = 1;
|
|
}
|
|
|
|
long long sx = 0, sy = 0, n = 0;
|
|
for (size_t i = 0; i < geom.size(); i++) {
|
|
if (geom[i].op == VT_MOVETO || geom[i].op == VT_LINETO) {
|
|
sx += geom[i].x;
|
|
sy += geom[i].y;
|
|
n++;
|
|
}
|
|
}
|
|
|
|
if (n > 0) {
|
|
sx /= n;
|
|
sy /= n;
|
|
|
|
drawvec out;
|
|
out.push_back(draw(VT_MOVETO, sx - (width / 2), sy - (height / 2)));
|
|
out.push_back(draw(VT_LINETO, sx - (width / 2) + width, sy - (height / 2)));
|
|
out.push_back(draw(VT_LINETO, sx - (width / 2) + width, sy - (height / 2) + height));
|
|
out.push_back(draw(VT_LINETO, sx - (width / 2), sy - (height / 2) + height));
|
|
out.push_back(draw(VT_LINETO, sx - (width / 2), sy - (height / 2)));
|
|
|
|
return out;
|
|
} else {
|
|
return drawvec();
|
|
}
|
|
}
|
|
|
|
double simplify_partial(partial *p, drawvec &shared_nodes) {
|
|
drawvec geom;
|
|
|
|
for (size_t j = 0; j < p->geoms.size(); j++) {
|
|
for (size_t k = 0; k < p->geoms[j].size(); k++) {
|
|
geom.push_back(p->geoms[j][k]);
|
|
}
|
|
}
|
|
|
|
p->geoms.clear(); // avoid keeping two copies in memory
|
|
signed char t = p->t;
|
|
int z = p->z;
|
|
int line_detail = p->line_detail;
|
|
int maxzoom = p->maxzoom;
|
|
|
|
if (additional[A_GRID_LOW_ZOOMS] && z < maxzoom) {
|
|
geom = stairstep(geom, z, line_detail);
|
|
}
|
|
|
|
double area = 0;
|
|
if (t == VT_POLYGON) {
|
|
area = get_mp_area(geom);
|
|
}
|
|
|
|
if ((t == VT_LINE || t == VT_POLYGON) && !(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]) || (z < maxzoom && additional[A_GRID_LOW_ZOOMS]))) {
|
|
// Now I finally remember why it doesn't simplify if the feature was reduced:
|
|
// because it makes square placeholders look like weird triangular placeholders.
|
|
// Only matters if simplification is set higher than the tiny polygon size.
|
|
// Tiny polygons that are part of a tiny multipolygon will still get simplified.
|
|
if (!p->reduced) {
|
|
if (t == VT_LINE) {
|
|
// continues to deduplicate to line_detail even if we have extra detail
|
|
geom = remove_noop(geom, t, 32 - z - line_detail);
|
|
}
|
|
|
|
bool already_marked = false;
|
|
if (additional[A_DETECT_SHARED_BORDERS] && t == VT_POLYGON) {
|
|
already_marked = true;
|
|
}
|
|
|
|
if (!already_marked) {
|
|
if (p->coalesced && t == VT_POLYGON) {
|
|
// clean coalesced polygons before simplification to avoid
|
|
// introducing shards between shapes that otherwise would have
|
|
// unioned exactly
|
|
geom = clean_or_clip_poly(geom, 0, 0, false);
|
|
}
|
|
|
|
// continues to simplify to line_detail even if we have extra detail
|
|
drawvec ngeom = simplify_lines(geom, z, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), p->simplification, t == VT_POLYGON ? 4 : 0, shared_nodes);
|
|
|
|
if (t != VT_POLYGON || ngeom.size() >= 3) {
|
|
geom = ngeom;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (t == VT_LINE && additional[A_REVERSE]) {
|
|
geom = reorder_lines(geom);
|
|
}
|
|
|
|
p->geoms.push_back(geom);
|
|
return area;
|
|
}
|
|
|
|
void *partial_feature_worker(void *v) {
|
|
struct partial_arg *a = (struct partial_arg *) v;
|
|
std::vector<struct partial> *partials = a->partials;
|
|
|
|
for (size_t i = a->task; i < (*partials).size(); i += a->tasks) {
|
|
double area = simplify_partial(&((*partials)[i]), *(a->shared_nodes));
|
|
|
|
signed char t = (*partials)[i].t;
|
|
int z = (*partials)[i].z;
|
|
int out_detail = (*partials)[i].extra_detail;
|
|
|
|
drawvec geom = (*partials)[i].geoms[0];
|
|
to_tile_scale(geom, z, out_detail);
|
|
|
|
if (t == VT_POLYGON) {
|
|
// Scaling may have made the polygon degenerate.
|
|
// Give Clipper a chance to try to fix it.
|
|
{
|
|
drawvec before = geom;
|
|
geom = clean_or_clip_poly(geom, 0, 0, false);
|
|
if (additional[A_DEBUG_POLYGON]) {
|
|
check_polygon(geom);
|
|
}
|
|
|
|
if (geom.size() < 3) {
|
|
if (area > 0) {
|
|
// area is in world coordinates, calculated before scaling down
|
|
geom = revive_polygon(before, area, z, out_detail);
|
|
} else {
|
|
geom.clear();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (t == VT_POLYGON && additional[A_GENERATE_POLYGON_LABEL_POINTS]) {
|
|
t = (*partials)[i].t = VT_POINT;
|
|
geom = checkerboard_anchors(from_tile_scale(geom, z, out_detail), (*partials)[i].tx, (*partials)[i].ty, z, (*partials)[i].label_point);
|
|
to_tile_scale(geom, z, out_detail);
|
|
}
|
|
|
|
(*partials)[i].index = i;
|
|
|
|
std::vector<drawvec> geoms; // artifact of former polygon-splitting to reduce geometric complexity
|
|
geoms.push_back(geom);
|
|
(*partials)[i].geoms = geoms;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap) {
|
|
if (gamma > 0) {
|
|
if (*gap > 0) {
|
|
if (index == *previndex) {
|
|
return 1; // Exact duplicate: can't fulfil the gap requirement
|
|
}
|
|
|
|
if (index < *previndex || std::exp(std::log((index - *previndex) / scale) * gamma) >= *gap) {
|
|
// Dot is further from the previous than the nth root of the gap,
|
|
// so produce it, and choose a new gap at the next point.
|
|
*gap = 0;
|
|
} else {
|
|
return 1;
|
|
}
|
|
} else if (index >= *previndex) {
|
|
*gap = (index - *previndex) / scale;
|
|
|
|
if (*gap == 0) {
|
|
return 1; // Exact duplicate: skip
|
|
} else if (*gap < 1) {
|
|
return 1; // Narrow dot spacing: need to stretch out
|
|
} else {
|
|
*gap = 0; // Wider spacing than minimum: so pass through unchanged
|
|
}
|
|
}
|
|
|
|
*previndex = index;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
// Does not fix up moveto/lineto
|
|
static drawvec reverse_subring(drawvec const &dv) {
|
|
drawvec out;
|
|
|
|
for (size_t i = dv.size(); i > 0; i--) {
|
|
out.push_back(dv[i - 1]);
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
struct edge {
|
|
unsigned x1 = 0;
|
|
unsigned y1 = 0;
|
|
unsigned x2 = 0;
|
|
unsigned y2 = 0;
|
|
unsigned ring = 0;
|
|
|
|
edge(unsigned _x1, unsigned _y1, unsigned _x2, unsigned _y2, unsigned _ring) {
|
|
x1 = _x1;
|
|
y1 = _y1;
|
|
x2 = _x2;
|
|
y2 = _y2;
|
|
ring = _ring;
|
|
}
|
|
|
|
bool operator<(const edge &s) const {
|
|
long long cmp = (long long) y1 - s.y1;
|
|
if (cmp == 0) {
|
|
cmp = (long long) x1 - s.x1;
|
|
}
|
|
if (cmp == 0) {
|
|
cmp = (long long) y2 - s.y2;
|
|
}
|
|
if (cmp == 0) {
|
|
cmp = (long long) x2 - s.x2;
|
|
}
|
|
return cmp < 0;
|
|
}
|
|
};
|
|
|
|
struct edgecmp_ring {
|
|
bool operator()(const edge &a, const edge &b) {
|
|
long long cmp = (long long) a.y1 - b.y1;
|
|
if (cmp == 0) {
|
|
cmp = (long long) a.x1 - b.x1;
|
|
}
|
|
if (cmp == 0) {
|
|
cmp = (long long) a.y2 - b.y2;
|
|
}
|
|
if (cmp == 0) {
|
|
cmp = (long long) a.x2 - b.x2;
|
|
}
|
|
if (cmp == 0) {
|
|
cmp = (long long) a.ring - b.ring;
|
|
}
|
|
return cmp < 0;
|
|
}
|
|
} edgecmp_ring;
|
|
|
|
bool edges_same(std::pair<std::vector<edge>::iterator, std::vector<edge>::iterator> e1, std::pair<std::vector<edge>::iterator, std::vector<edge>::iterator> e2) {
|
|
if ((e2.second - e2.first) != (e1.second - e1.first)) {
|
|
return false;
|
|
}
|
|
|
|
while (e1.first != e1.second) {
|
|
if (e1.first->ring != e2.first->ring) {
|
|
return false;
|
|
}
|
|
|
|
++e1.first;
|
|
++e2.first;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool find_common_edges(std::vector<partial> &partials, int z, int line_detail, double simplification, int maxzoom, double merge_fraction) {
|
|
size_t merge_count = ceil((1 - merge_fraction) * partials.size());
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
if (partials[i].t == VT_POLYGON) {
|
|
for (size_t j = 0; j < partials[i].geoms.size(); j++) {
|
|
drawvec &g = partials[i].geoms[j];
|
|
drawvec out;
|
|
|
|
for (size_t k = 0; k < g.size(); k++) {
|
|
if (g[k].op == VT_LINETO && k > 0 && g[k - 1] == g[k]) {
|
|
;
|
|
} else {
|
|
out.push_back(g[k]);
|
|
}
|
|
}
|
|
|
|
partials[i].geoms[j] = out;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Construct a mapping from all polygon edges to the set of rings
|
|
// that each edge appears in. (The ring number is across all polygons;
|
|
// we don't need to look it back up, just to tell where it changes.)
|
|
|
|
std::vector<edge> edges;
|
|
size_t ring = 0;
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
if (partials[i].t == VT_POLYGON) {
|
|
for (size_t j = 0; j < partials[i].geoms.size(); j++) {
|
|
for (size_t k = 0; k + 1 < partials[i].geoms[j].size(); k++) {
|
|
if (partials[i].geoms[j][k].op == VT_MOVETO) {
|
|
ring++;
|
|
}
|
|
|
|
if (partials[i].geoms[j][k + 1].op == VT_LINETO) {
|
|
drawvec dv;
|
|
if (partials[i].geoms[j][k] < partials[i].geoms[j][k + 1]) {
|
|
dv.push_back(partials[i].geoms[j][k]);
|
|
dv.push_back(partials[i].geoms[j][k + 1]);
|
|
} else {
|
|
dv.push_back(partials[i].geoms[j][k + 1]);
|
|
dv.push_back(partials[i].geoms[j][k]);
|
|
}
|
|
|
|
edges.push_back(edge(dv[0].x, dv[0].y, dv[1].x, dv[1].y, ring));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::sort(edges.begin(), edges.end(), edgecmp_ring);
|
|
std::set<draw> necessaries;
|
|
|
|
// Now mark all the points where the set of rings using the edge on one side
|
|
// is not the same as the set of rings using the edge on the other side.
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
if (partials[i].t == VT_POLYGON) {
|
|
for (size_t j = 0; j < partials[i].geoms.size(); j++) {
|
|
drawvec &g = partials[i].geoms[j];
|
|
|
|
for (size_t k = 0; k < g.size(); k++) {
|
|
g[k].necessary = 0;
|
|
}
|
|
|
|
for (size_t a = 0; a < g.size(); a++) {
|
|
if (g[a].op == VT_MOVETO) {
|
|
size_t b;
|
|
|
|
for (b = a + 1; b < g.size(); b++) {
|
|
if (g[b].op != VT_LINETO) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
// -1 because of duplication at the end
|
|
size_t s = b - a - 1;
|
|
|
|
if (s > 0) {
|
|
drawvec left;
|
|
if (g[a + (s - 1) % s] < g[a]) {
|
|
left.push_back(g[a + (s - 1) % s]);
|
|
left.push_back(g[a]);
|
|
} else {
|
|
left.push_back(g[a]);
|
|
left.push_back(g[a + (s - 1) % s]);
|
|
}
|
|
if (left[1] < left[0]) {
|
|
fprintf(stderr, "left misordered\n");
|
|
}
|
|
std::pair<std::vector<edge>::iterator, std::vector<edge>::iterator> e1 = std::equal_range(edges.begin(), edges.end(), edge(left[0].x, left[0].y, left[1].x, left[1].y, 0));
|
|
|
|
for (size_t k = 0; k < s; k++) {
|
|
drawvec right;
|
|
|
|
if (g[a + k] < g[a + k + 1]) {
|
|
right.push_back(g[a + k]);
|
|
right.push_back(g[a + k + 1]);
|
|
} else {
|
|
right.push_back(g[a + k + 1]);
|
|
right.push_back(g[a + k]);
|
|
}
|
|
|
|
std::pair<std::vector<edge>::iterator, std::vector<edge>::iterator> e2 = std::equal_range(edges.begin(), edges.end(), edge(right[0].x, right[0].y, right[1].x, right[1].y, 0));
|
|
|
|
if (right[1] < right[0]) {
|
|
fprintf(stderr, "left misordered\n");
|
|
}
|
|
|
|
if (e1.first == e1.second || e2.first == e2.second) {
|
|
fprintf(stderr, "Internal error: polygon edge lookup failed for %lld,%lld to %lld,%lld or %lld,%lld to %lld,%lld\n", left[0].x, left[0].y, left[1].x, left[1].y, right[0].x, right[0].y, right[1].x, right[1].y);
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
|
|
if (!edges_same(e1, e2)) {
|
|
g[a + k].necessary = 1;
|
|
necessaries.insert(g[a + k]);
|
|
}
|
|
|
|
e1 = e2;
|
|
}
|
|
}
|
|
|
|
a = b - 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
edges.clear();
|
|
std::map<drawvec, size_t> arcs;
|
|
std::multimap<ssize_t, size_t> merge_candidates; // from arc to partial
|
|
|
|
// Roll rings that include a necessary point around so they start at one
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
if (partials[i].t == VT_POLYGON) {
|
|
for (size_t j = 0; j < partials[i].geoms.size(); j++) {
|
|
drawvec &g = partials[i].geoms[j];
|
|
|
|
for (size_t k = 0; k < g.size(); k++) {
|
|
if (necessaries.count(g[k]) != 0) {
|
|
g[k].necessary = 1;
|
|
}
|
|
}
|
|
|
|
for (size_t k = 0; k < g.size(); k++) {
|
|
if (g[k].op == VT_MOVETO) {
|
|
ssize_t necessary = -1;
|
|
ssize_t lowest = k;
|
|
size_t l;
|
|
for (l = k + 1; l < g.size(); l++) {
|
|
if (g[l].op != VT_LINETO) {
|
|
break;
|
|
}
|
|
|
|
if (g[l].necessary) {
|
|
necessary = l;
|
|
}
|
|
if (g[l] < g[lowest]) {
|
|
lowest = l;
|
|
}
|
|
}
|
|
|
|
if (necessary < 0) {
|
|
necessary = lowest;
|
|
// Add a necessary marker if there was none in the ring,
|
|
// so the arc code below can find it.
|
|
g[lowest].necessary = 1;
|
|
}
|
|
|
|
{
|
|
drawvec tmp;
|
|
|
|
// l - 1 because the endpoint is duplicated
|
|
for (size_t m = necessary; m < l - 1; m++) {
|
|
tmp.push_back(g[m]);
|
|
}
|
|
for (ssize_t m = k; m < necessary; m++) {
|
|
tmp.push_back(g[m]);
|
|
}
|
|
|
|
// replace the endpoint
|
|
tmp.push_back(g[necessary]);
|
|
|
|
if (tmp.size() != l - k) {
|
|
fprintf(stderr, "internal error shifting ring\n");
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
|
|
for (size_t m = 0; m < tmp.size(); m++) {
|
|
if (m == 0) {
|
|
tmp[m].op = VT_MOVETO;
|
|
} else {
|
|
tmp[m].op = VT_LINETO;
|
|
}
|
|
|
|
g[k + m] = tmp[m];
|
|
}
|
|
}
|
|
|
|
// Now peel off each set of segments from one necessary point to the next
|
|
// into an "arc" as in TopoJSON
|
|
|
|
for (size_t m = k; m < l; m++) {
|
|
if (!g[m].necessary) {
|
|
fprintf(stderr, "internal error in arc building\n");
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
|
|
drawvec arc;
|
|
size_t n;
|
|
for (n = m; n < l; n++) {
|
|
arc.push_back(g[n]);
|
|
if (n > m && g[n].necessary) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
auto f = arcs.find(arc);
|
|
if (f == arcs.end()) {
|
|
drawvec arc2 = reverse_subring(arc);
|
|
|
|
auto f2 = arcs.find(arc2);
|
|
if (f2 == arcs.end()) {
|
|
// Add new arc
|
|
size_t added = arcs.size() + 1;
|
|
arcs.insert(std::pair<drawvec, size_t>(arc, added));
|
|
partials[i].arc_polygon.push_back(added);
|
|
merge_candidates.insert(std::pair<ssize_t, size_t>(added, i));
|
|
} else {
|
|
partials[i].arc_polygon.push_back(-(ssize_t) f2->second);
|
|
merge_candidates.insert(std::pair<ssize_t, size_t>(-(ssize_t) f2->second, i));
|
|
}
|
|
} else {
|
|
partials[i].arc_polygon.push_back(f->second);
|
|
merge_candidates.insert(std::pair<ssize_t, size_t>(f->second, i));
|
|
}
|
|
|
|
m = n - 1;
|
|
}
|
|
|
|
partials[i].arc_polygon.push_back(0);
|
|
|
|
k = l - 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Simplify each arc
|
|
|
|
std::vector<drawvec> simplified_arcs;
|
|
|
|
size_t count = 0;
|
|
for (auto ai = arcs.begin(); ai != arcs.end(); ++ai) {
|
|
if (simplified_arcs.size() < ai->second + 1) {
|
|
simplified_arcs.resize(ai->second + 1);
|
|
}
|
|
|
|
drawvec dv = ai->first;
|
|
for (size_t i = 0; i < dv.size(); i++) {
|
|
if (i == 0) {
|
|
dv[i].op = VT_MOVETO;
|
|
} else {
|
|
dv[i].op = VT_LINETO;
|
|
}
|
|
}
|
|
if (!(prevent[P_SIMPLIFY] || (z == maxzoom && prevent[P_SIMPLIFY_LOW]) || (z < maxzoom && additional[A_GRID_LOW_ZOOMS]))) {
|
|
simplified_arcs[ai->second] = simplify_lines(dv, z, line_detail, !(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, 4, drawvec());
|
|
} else {
|
|
simplified_arcs[ai->second] = dv;
|
|
}
|
|
count++;
|
|
}
|
|
|
|
// If necessary, merge some adjacent polygons into some other polygons
|
|
|
|
struct merge_order {
|
|
ssize_t edge = 0;
|
|
unsigned long long gap = 0;
|
|
size_t p1 = 0;
|
|
size_t p2 = 0;
|
|
|
|
bool operator<(const merge_order &m) const {
|
|
return gap < m.gap;
|
|
}
|
|
};
|
|
std::vector<merge_order> order;
|
|
|
|
for (ssize_t i = 0; i < (ssize_t) simplified_arcs.size(); i++) {
|
|
auto r1 = merge_candidates.equal_range(i);
|
|
for (auto r1i = r1.first; r1i != r1.second; ++r1i) {
|
|
auto r2 = merge_candidates.equal_range(-i);
|
|
for (auto r2i = r2.first; r2i != r2.second; ++r2i) {
|
|
if (r1i->second != r2i->second) {
|
|
merge_order mo;
|
|
mo.edge = i;
|
|
if (partials[r1i->second].index > partials[r2i->second].index) {
|
|
mo.gap = partials[r1i->second].index - partials[r2i->second].index;
|
|
} else {
|
|
mo.gap = partials[r2i->second].index - partials[r1i->second].index;
|
|
}
|
|
mo.p1 = r1i->second;
|
|
mo.p2 = r2i->second;
|
|
order.push_back(mo);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
std::sort(order.begin(), order.end());
|
|
|
|
size_t merged = 0;
|
|
for (size_t o = 0; o < order.size(); o++) {
|
|
if (merged >= merge_count) {
|
|
break;
|
|
}
|
|
|
|
size_t i = order[o].p1;
|
|
while (partials[i].renamed >= 0) {
|
|
i = partials[i].renamed;
|
|
}
|
|
size_t i2 = order[o].p2;
|
|
while (partials[i2].renamed >= 0) {
|
|
i2 = partials[i2].renamed;
|
|
}
|
|
|
|
for (size_t j = 0; j < partials[i].arc_polygon.size() && merged < merge_count; j++) {
|
|
if (partials[i].arc_polygon[j] == order[o].edge) {
|
|
{
|
|
// XXX snap links
|
|
if (partials[order[o].p2].arc_polygon.size() > 0) {
|
|
// This has to merge the ring that contains the anti-arc to this arc
|
|
// into the current ring, and then add whatever other rings were in
|
|
// that feature on to the end.
|
|
//
|
|
// This can't be good for keeping parent-child relationships among
|
|
// the rings in order, but Wagyu should sort that out later
|
|
|
|
std::vector<ssize_t> additions;
|
|
std::vector<ssize_t> &here = partials[i].arc_polygon;
|
|
std::vector<ssize_t> &other = partials[i2].arc_polygon;
|
|
|
|
#if 0
|
|
printf("seeking %zd\n", partials[i].arc_polygon[j]);
|
|
printf("before: ");
|
|
for (size_t k = 0; k < here.size(); k++) {
|
|
printf("%zd ", here[k]);
|
|
}
|
|
printf("\n");
|
|
printf("other: ");
|
|
for (size_t k = 0; k < other.size(); k++) {
|
|
printf("%zd ", other[k]);
|
|
}
|
|
printf("\n");
|
|
#endif
|
|
|
|
for (size_t k = 0; k < other.size(); k++) {
|
|
size_t l;
|
|
for (l = k; l < other.size(); l++) {
|
|
if (other[l] == 0) {
|
|
break;
|
|
}
|
|
}
|
|
if (l >= other.size()) {
|
|
l--;
|
|
}
|
|
|
|
#if 0
|
|
for (size_t m = k; m <= l; m++) {
|
|
printf("%zd ", other[m]);
|
|
}
|
|
printf("\n");
|
|
#endif
|
|
|
|
size_t m;
|
|
for (m = k; m <= l; m++) {
|
|
if (other[m] == -partials[i].arc_polygon[j]) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (m <= l) {
|
|
// Found the shared arc
|
|
|
|
here.erase(here.begin() + j);
|
|
|
|
size_t off = 0;
|
|
for (size_t n = m + 1; n < l; n++) {
|
|
here.insert(here.begin() + j + off, other[n]);
|
|
off++;
|
|
}
|
|
for (size_t n = k; n < m; n++) {
|
|
here.insert(here.begin() + j + off, other[n]);
|
|
off++;
|
|
}
|
|
} else {
|
|
// Looking at some other ring
|
|
|
|
for (size_t n = k; n <= l; n++) {
|
|
additions.push_back(other[n]);
|
|
}
|
|
}
|
|
|
|
k = l;
|
|
}
|
|
|
|
partials[i2].arc_polygon.clear();
|
|
partials[i2].renamed = i;
|
|
merged++;
|
|
|
|
for (size_t k = 0; k < additions.size(); k++) {
|
|
partials[i].arc_polygon.push_back(additions[k]);
|
|
}
|
|
|
|
#if 0
|
|
printf("after: ");
|
|
for (size_t k = 0; k < here.size(); k++) {
|
|
printf("%zd ", here[k]);
|
|
}
|
|
printf("\n");
|
|
#endif
|
|
|
|
#if 0
|
|
for (size_t k = 0; k + 1 < here.size(); k++) {
|
|
if (here[k] != 0 && here[k + 1] != 0) {
|
|
if (simplified_arcs[here[k + 1]][0] != simplified_arcs[here[k]][simplified_arcs[here[k]].size() - 1]) {
|
|
printf("error from %zd to %zd\n", here[k], here[k + 1]);
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Turn the arc representations of the polygons back into standard polygon geometries
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
if (partials[i].t == VT_POLYGON) {
|
|
partials[i].geoms.resize(0);
|
|
partials[i].geoms.push_back(drawvec());
|
|
bool at_start = true;
|
|
draw first(-1, 0, 0);
|
|
|
|
for (size_t j = 0; j < partials[i].arc_polygon.size(); j++) {
|
|
ssize_t p = partials[i].arc_polygon[j];
|
|
|
|
if (p == 0) {
|
|
if (first.op >= 0) {
|
|
partials[i].geoms[0].push_back(first);
|
|
first = draw(-1, 0, 0);
|
|
}
|
|
at_start = true;
|
|
} else if (p > 0) {
|
|
for (size_t k = 0; k + 1 < simplified_arcs[p].size(); k++) {
|
|
if (at_start) {
|
|
partials[i].geoms[0].push_back(draw(VT_MOVETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y));
|
|
first = draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y);
|
|
} else {
|
|
partials[i].geoms[0].push_back(draw(VT_LINETO, simplified_arcs[p][k].x, simplified_arcs[p][k].y));
|
|
}
|
|
at_start = 0;
|
|
}
|
|
} else { /* p < 0 */
|
|
for (ssize_t k = simplified_arcs[-p].size() - 1; k > 0; k--) {
|
|
if (at_start) {
|
|
partials[i].geoms[0].push_back(draw(VT_MOVETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y));
|
|
first = draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y);
|
|
} else {
|
|
partials[i].geoms[0].push_back(draw(VT_LINETO, simplified_arcs[-p][k].x, simplified_arcs[-p][k].y));
|
|
}
|
|
at_start = 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (merged >= merge_count) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
unsigned long long choose_mingap(std::vector<unsigned long long> const &indices, double f) {
|
|
unsigned long long bot = ULLONG_MAX;
|
|
unsigned long long top = 0;
|
|
|
|
for (size_t i = 0; i < indices.size(); i++) {
|
|
if (i > 0 && indices[i] >= indices[i - 1]) {
|
|
if (indices[i] - indices[i - 1] > top) {
|
|
top = indices[i] - indices[i - 1];
|
|
}
|
|
if (indices[i] - indices[i - 1] < bot) {
|
|
bot = indices[i] - indices[i - 1];
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t want = indices.size() * f;
|
|
while (top - bot > 2) {
|
|
unsigned long long guess = bot / 2 + top / 2;
|
|
size_t count = 0;
|
|
unsigned long long prev = 0;
|
|
|
|
for (size_t i = 0; i < indices.size(); i++) {
|
|
if (indices[i] - prev >= guess) {
|
|
count++;
|
|
prev = indices[i];
|
|
}
|
|
}
|
|
|
|
if (count > want) {
|
|
bot = guess;
|
|
} else if (count < want) {
|
|
top = guess;
|
|
} else {
|
|
return guess;
|
|
}
|
|
}
|
|
|
|
return top;
|
|
}
|
|
|
|
long long choose_minextent(std::vector<long long> &extents, double f) {
|
|
std::sort(extents.begin(), extents.end());
|
|
return extents[(extents.size() - 1) * (1 - f)];
|
|
}
|
|
|
|
struct write_tile_args {
|
|
struct task *tasks = NULL;
|
|
char *stringpool = NULL;
|
|
int min_detail = 0;
|
|
sqlite3 *outdb = NULL;
|
|
const char *outdir = NULL;
|
|
int buffer = 0;
|
|
const char *fname = NULL;
|
|
FILE **geomfile = NULL;
|
|
double todo = 0;
|
|
std::atomic<long long> *along = NULL;
|
|
double gamma = 0;
|
|
double gamma_out = 0;
|
|
int child_shards = 0;
|
|
int *geomfd = NULL;
|
|
off_t *geom_size = NULL;
|
|
std::atomic<unsigned> *midx = NULL;
|
|
std::atomic<unsigned> *midy = NULL;
|
|
int maxzoom = 0;
|
|
int minzoom = 0;
|
|
int full_detail = 0;
|
|
int low_detail = 0;
|
|
double simplification = 0;
|
|
std::atomic<long long> *most = NULL;
|
|
long long *pool_off = NULL;
|
|
unsigned *initial_x = NULL;
|
|
unsigned *initial_y = NULL;
|
|
std::atomic<int> *running = NULL;
|
|
int err = 0;
|
|
std::vector<std::map<std::string, layermap_entry>> *layermaps = NULL;
|
|
std::vector<std::vector<std::string>> *layer_unmaps = NULL;
|
|
size_t pass = 0;
|
|
unsigned long long mingap = 0;
|
|
unsigned long long mingap_out = 0;
|
|
long long minextent = 0;
|
|
long long minextent_out = 0;
|
|
double fraction = 0;
|
|
double fraction_out = 0;
|
|
size_t tile_size_out = 0;
|
|
size_t feature_count_out = 0;
|
|
const char *prefilter = NULL;
|
|
const char *postfilter = NULL;
|
|
std::map<std::string, attribute_op> const *attribute_accum = NULL;
|
|
bool still_dropping = false;
|
|
int wrote_zoom = 0;
|
|
size_t tiling_seg = 0;
|
|
struct json_object *filter = NULL;
|
|
std::atomic<size_t> *dropped_count = NULL;
|
|
atomic_strategy *strategy = NULL;
|
|
};
|
|
|
|
bool clip_to_tile(serial_feature &sf, int z, long long buffer) {
|
|
int quick = quick_check(sf.bbox, z, buffer);
|
|
|
|
if (z == 0) {
|
|
if (sf.bbox[0] <= (1LL << 32) * buffer / 256 || sf.bbox[2] >= (1LL << 32) - ((1LL << 32) * buffer / 256)) {
|
|
// If the geometry extends off the edge of the world, concatenate on another copy
|
|
// shifted by 360 degrees, and then make sure both copies get clipped down to size.
|
|
|
|
size_t n = sf.geometry.size();
|
|
|
|
if (sf.bbox[0] <= (1LL << 32) * buffer / 256) {
|
|
for (size_t i = 0; i < n; i++) {
|
|
sf.geometry.push_back(draw(sf.geometry[i].op, sf.geometry[i].x + (1LL << 32), sf.geometry[i].y));
|
|
}
|
|
}
|
|
|
|
if (sf.bbox[2] >= (1LL << 32) - ((1LL << 32) * buffer / 256)) {
|
|
for (size_t i = 0; i < n; i++) {
|
|
sf.geometry.push_back(draw(sf.geometry[i].op, sf.geometry[i].x - (1LL << 32), sf.geometry[i].y));
|
|
}
|
|
}
|
|
|
|
sf.bbox[0] = 0;
|
|
sf.bbox[2] = 1LL << 32;
|
|
|
|
quick = -1;
|
|
}
|
|
}
|
|
|
|
if (quick == 0) {
|
|
return true;
|
|
}
|
|
|
|
// Can't accept the quick check if guaranteeing no duplication, since the
|
|
// overlap might have been in the buffer.
|
|
if (quick != 1 || prevent[P_DUPLICATION]) {
|
|
drawvec clipped;
|
|
|
|
// Do the clipping, even if we are going to include the whole feature,
|
|
// so that we can know whether the feature itself, or only the feature's
|
|
// bounding box, touches the tile.
|
|
|
|
if (sf.t == VT_LINE) {
|
|
clipped = clip_lines(sf.geometry, z, buffer);
|
|
}
|
|
if (sf.t == VT_POLYGON) {
|
|
clipped = simple_clip_poly(sf.geometry, z, buffer);
|
|
}
|
|
if (sf.t == VT_POINT) {
|
|
clipped = clip_point(sf.geometry, z, buffer);
|
|
}
|
|
|
|
clipped = remove_noop(clipped, sf.t, 0);
|
|
|
|
// Must clip at z0 even if we don't want clipping, to handle features
|
|
// that are duplicated across the date line
|
|
|
|
if (prevent[P_DUPLICATION] && z != 0) {
|
|
if (point_within_tile((sf.bbox[0] + sf.bbox[2]) / 2, (sf.bbox[1] + sf.bbox[3]) / 2, z)) {
|
|
// sf.geometry is unchanged
|
|
} else {
|
|
sf.geometry.clear();
|
|
}
|
|
} else if (prevent[P_CLIPPING] && z != 0) {
|
|
if (clipped.size() == 0) {
|
|
sf.geometry.clear();
|
|
} else {
|
|
// sf.geometry is unchanged
|
|
}
|
|
} else {
|
|
sf.geometry = clipped;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
void remove_attributes(serial_feature &sf, std::set<std::string> const &exclude_attributes, const char *stringpool, long long *pool_off) {
|
|
for (ssize_t i = sf.keys.size() - 1; i >= 0; i--) {
|
|
std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1;
|
|
if (exclude_attributes.count(key) > 0) {
|
|
sf.keys.erase(sf.keys.begin() + i);
|
|
sf.values.erase(sf.values.begin() + i);
|
|
}
|
|
}
|
|
|
|
for (ssize_t i = sf.full_keys.size() - 1; i >= 0; i--) {
|
|
std::string key = sf.full_keys[i];
|
|
if (exclude_attributes.count(key) > 0) {
|
|
sf.full_keys.erase(sf.full_keys.begin() + i);
|
|
sf.full_values.erase(sf.full_values.begin() + i);
|
|
}
|
|
}
|
|
}
|
|
|
|
serial_feature next_feature(FILE *geoms, std::atomic<long long> *geompos_in, int z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y, long long *original_features, long long *unclipped_features, int nextzoom, int maxzoom, int minzoom, int max_zoom_increment, size_t pass, std::atomic<long long> *along, long long alongminus, int buffer, int *within, FILE **geomfile, std::atomic<long long> *geompos, std::atomic<double> *oprogress, double todo, const char *fname, int child_shards, struct json_object *filter, const char *stringpool, long long *pool_off, std::vector<std::vector<std::string>> *layer_unmaps, bool first_time) {
|
|
while (1) {
|
|
serial_feature sf = deserialize_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y);
|
|
if (sf.t < 0) {
|
|
return sf;
|
|
}
|
|
|
|
size_t passes = pass + 1;
|
|
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10;
|
|
if (progress >= *oprogress + 0.1) {
|
|
if (!quiet && !quiet_progress && progress_time()) {
|
|
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
|
|
}
|
|
if (logger.json_enabled && progress_time()) {
|
|
logger.progress_tile(progress);
|
|
}
|
|
*oprogress = progress;
|
|
}
|
|
|
|
(*original_features)++;
|
|
|
|
if (clip_to_tile(sf, z, buffer)) {
|
|
continue;
|
|
}
|
|
|
|
if (sf.geometry.size() > 0) {
|
|
(*unclipped_features)++;
|
|
}
|
|
|
|
if (first_time && pass == 0) { /* only write out the next zoom once, even if we retry */
|
|
if (sf.tippecanoe_maxzoom == -1 || sf.tippecanoe_maxzoom >= nextzoom) {
|
|
rewrite(sf.geometry, z, nextzoom, maxzoom, sf.bbox, tx, ty, buffer, within, geompos, geomfile, fname, sf.t, sf.layer, sf.feature_minzoom, child_shards, max_zoom_increment, sf.seq, sf.tippecanoe_minzoom, sf.tippecanoe_maxzoom, sf.segment, initial_x, initial_y, sf.keys, sf.values, sf.has_id, sf.id, sf.index, sf.label_point, sf.extent);
|
|
}
|
|
}
|
|
|
|
if (z < minzoom) {
|
|
continue;
|
|
}
|
|
|
|
if (sf.tippecanoe_minzoom != -1 && z < sf.tippecanoe_minzoom) {
|
|
continue;
|
|
}
|
|
if (sf.tippecanoe_maxzoom != -1 && z > sf.tippecanoe_maxzoom) {
|
|
continue;
|
|
}
|
|
|
|
if (filter != NULL) {
|
|
std::map<std::string, mvt_value> attributes;
|
|
std::string layername = (*layer_unmaps)[sf.segment][sf.layer];
|
|
std::set<std::string> exclude_attributes;
|
|
|
|
for (size_t i = 0; i < sf.keys.size(); i++) {
|
|
std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1;
|
|
|
|
serial_val sv;
|
|
sv.type = (stringpool + pool_off[sf.segment])[sf.values[i]];
|
|
sv.s = stringpool + pool_off[sf.segment] + sf.values[i] + 1;
|
|
|
|
mvt_value val = stringified_to_mvt_value(sv.type, sv.s.c_str());
|
|
attributes.insert(std::pair<std::string, mvt_value>(key, val));
|
|
}
|
|
|
|
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
|
std::string key = sf.full_keys[i];
|
|
mvt_value val = stringified_to_mvt_value(sf.full_values[i].type, sf.full_values[i].s.c_str());
|
|
|
|
attributes.insert(std::pair<std::string, mvt_value>(key, val));
|
|
}
|
|
|
|
if (sf.has_id) {
|
|
mvt_value v;
|
|
v.type = mvt_uint;
|
|
v.numeric_value.uint_value = sf.id;
|
|
|
|
attributes.insert(std::pair<std::string, mvt_value>("$id", v));
|
|
}
|
|
|
|
mvt_value v;
|
|
v.type = mvt_string;
|
|
|
|
if (sf.t == mvt_point) {
|
|
v.string_value = "Point";
|
|
} else if (sf.t == mvt_linestring) {
|
|
v.string_value = "LineString";
|
|
} else if (sf.t == mvt_polygon) {
|
|
v.string_value = "Polygon";
|
|
}
|
|
|
|
attributes.insert(std::pair<std::string, mvt_value>("$type", v));
|
|
|
|
mvt_value v2;
|
|
v2.type = mvt_uint;
|
|
v2.numeric_value.uint_value = z;
|
|
|
|
attributes.insert(std::pair<std::string, mvt_value>("$zoom", v2));
|
|
|
|
if (!evaluate(attributes, layername, filter, exclude_attributes)) {
|
|
continue;
|
|
}
|
|
|
|
if (exclude_attributes.size() > 0) {
|
|
remove_attributes(sf, exclude_attributes, stringpool, pool_off);
|
|
}
|
|
}
|
|
|
|
if (sf.tippecanoe_minzoom == -1 && z < sf.feature_minzoom) {
|
|
sf.dropped = true;
|
|
}
|
|
|
|
// Remove nulls, now that the expression evaluation filter has run
|
|
|
|
for (ssize_t i = (ssize_t) sf.keys.size() - 1; i >= 0; i--) {
|
|
int type = (stringpool + pool_off[sf.segment])[sf.values[i]];
|
|
|
|
if (type == mvt_null) {
|
|
sf.keys.erase(sf.keys.begin() + i);
|
|
sf.values.erase(sf.values.begin() + i);
|
|
}
|
|
}
|
|
|
|
for (ssize_t i = (ssize_t) sf.full_keys.size() - 1; i >= 0; i--) {
|
|
if (sf.full_values[i].type == mvt_null) {
|
|
sf.full_keys.erase(sf.full_keys.begin() + i);
|
|
sf.full_values.erase(sf.full_values.begin() + i);
|
|
}
|
|
}
|
|
|
|
return sf;
|
|
}
|
|
}
|
|
|
|
struct run_prefilter_args {
|
|
FILE *geoms = NULL;
|
|
std::atomic<long long> *geompos_in = NULL;
|
|
int z = 0;
|
|
unsigned tx = 0;
|
|
unsigned ty = 0;
|
|
unsigned *initial_x = 0;
|
|
unsigned *initial_y = 0;
|
|
long long *original_features = 0;
|
|
long long *unclipped_features = 0;
|
|
int nextzoom = 0;
|
|
int maxzoom = 0;
|
|
int minzoom = 0;
|
|
int max_zoom_increment = 0;
|
|
size_t pass = 0;
|
|
std::atomic<long long> *along = 0;
|
|
long long alongminus = 0;
|
|
int buffer = 0;
|
|
int *within = NULL;
|
|
FILE **geomfile = NULL;
|
|
std::atomic<long long> *geompos = NULL;
|
|
std::atomic<double> *oprogress = NULL;
|
|
double todo = 0;
|
|
const char *fname = 0;
|
|
int child_shards = 0;
|
|
std::vector<std::vector<std::string>> *layer_unmaps = NULL;
|
|
char *stringpool = NULL;
|
|
long long *pool_off = NULL;
|
|
FILE *prefilter_fp = NULL;
|
|
struct json_object *filter = NULL;
|
|
bool first_time = false;
|
|
};
|
|
|
|
void *run_prefilter(void *v) {
|
|
run_prefilter_args *rpa = (run_prefilter_args *) v;
|
|
json_writer state(rpa->prefilter_fp);
|
|
|
|
while (1) {
|
|
serial_feature sf = next_feature(rpa->geoms, rpa->geompos_in, rpa->z, rpa->tx, rpa->ty, rpa->initial_x, rpa->initial_y, rpa->original_features, rpa->unclipped_features, rpa->nextzoom, rpa->maxzoom, rpa->minzoom, rpa->max_zoom_increment, rpa->pass, rpa->along, rpa->alongminus, rpa->buffer, rpa->within, rpa->geomfile, rpa->geompos, rpa->oprogress, rpa->todo, rpa->fname, rpa->child_shards, rpa->filter, rpa->stringpool, rpa->pool_off, rpa->layer_unmaps, rpa->first_time);
|
|
if (sf.t < 0) {
|
|
break;
|
|
}
|
|
|
|
mvt_layer tmp_layer;
|
|
tmp_layer.extent = 1LL << 32;
|
|
tmp_layer.name = (*(rpa->layer_unmaps))[sf.segment][sf.layer];
|
|
|
|
if (sf.t == VT_POLYGON) {
|
|
sf.geometry = close_poly(sf.geometry);
|
|
}
|
|
|
|
mvt_feature tmp_feature;
|
|
tmp_feature.type = sf.t;
|
|
tmp_feature.geometry = to_feature(sf.geometry);
|
|
tmp_feature.id = sf.id;
|
|
tmp_feature.has_id = sf.has_id;
|
|
tmp_feature.dropped = sf.dropped;
|
|
|
|
// Offset from tile coordinates back to world coordinates
|
|
unsigned sx = 0, sy = 0;
|
|
if (rpa->z != 0) {
|
|
sx = rpa->tx << (32 - rpa->z);
|
|
sy = rpa->ty << (32 - rpa->z);
|
|
}
|
|
for (size_t i = 0; i < tmp_feature.geometry.size(); i++) {
|
|
tmp_feature.geometry[i].x += sx;
|
|
tmp_feature.geometry[i].y += sy;
|
|
}
|
|
|
|
decode_meta(sf.keys, sf.values, rpa->stringpool + rpa->pool_off[sf.segment], tmp_layer, tmp_feature);
|
|
tmp_layer.features.push_back(tmp_feature);
|
|
|
|
layer_to_geojson(tmp_layer, 0, 0, 0, false, true, false, true, sf.index, sf.seq, sf.extent, true, state, 0);
|
|
}
|
|
|
|
if (fclose(rpa->prefilter_fp) != 0) {
|
|
if (errno == EPIPE) {
|
|
static bool warned = false;
|
|
if (!warned) {
|
|
fprintf(stderr, "Warning: broken pipe in prefilter\n");
|
|
warned = true;
|
|
}
|
|
} else {
|
|
perror("fclose output to prefilter");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
void add_tilestats(std::string const &layername, int z, std::vector<std::map<std::string, layermap_entry>> *layermaps, size_t tiling_seg, std::vector<std::vector<std::string>> *layer_unmaps, std::string const &key, serial_val const &val) {
|
|
std::map<std::string, layermap_entry> &layermap = (*layermaps)[tiling_seg];
|
|
if (layermap.count(layername) == 0) {
|
|
layermap_entry lme = layermap_entry(layermap.size());
|
|
lme.minzoom = z;
|
|
lme.maxzoom = z;
|
|
lme.retain = 1;
|
|
|
|
layermap.insert(std::pair<std::string, layermap_entry>(layername, lme));
|
|
|
|
if (lme.id >= (*layer_unmaps)[tiling_seg].size()) {
|
|
(*layer_unmaps)[tiling_seg].resize(lme.id + 1);
|
|
(*layer_unmaps)[tiling_seg][lme.id] = layername;
|
|
}
|
|
}
|
|
auto fk = layermap.find(layername);
|
|
if (fk == layermap.end()) {
|
|
fprintf(stderr, "Internal error: layer %s not found\n", layername.c_str());
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
|
|
type_and_string attrib;
|
|
attrib.type = val.type;
|
|
attrib.string = val.s;
|
|
|
|
add_to_file_keys(fk->second.file_keys, key, attrib);
|
|
}
|
|
|
|
void preserve_attribute(attribute_op op, serial_feature &, char *stringpool, long long *pool_off, std::string &key, serial_val &val, partial &p) {
|
|
if (p.need_tilestats.count(key) == 0) {
|
|
p.need_tilestats.insert(key);
|
|
}
|
|
|
|
// If the feature being merged into has this key as a metadata reference,
|
|
// promote it to a full_key so it can be modified
|
|
|
|
for (size_t i = 0; i < p.keys.size(); i++) {
|
|
if (strcmp(key.c_str(), stringpool + pool_off[p.segment] + p.keys[i] + 1) == 0) {
|
|
serial_val sv;
|
|
sv.s = stringpool + pool_off[p.segment] + p.values[i] + 1;
|
|
sv.type = (stringpool + pool_off[p.segment])[p.values[i]];
|
|
|
|
p.full_keys.push_back(key);
|
|
p.full_values.push_back(sv);
|
|
|
|
p.keys.erase(p.keys.begin() + i);
|
|
p.values.erase(p.values.begin() + i);
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < p.full_keys.size(); i++) {
|
|
if (key == p.full_keys[i]) {
|
|
switch (op) {
|
|
case op_sum:
|
|
p.full_values[i].s = milo::dtoa_milo(atof(p.full_values[i].s.c_str()) + atof(val.s.c_str()));
|
|
p.full_values[i].type = mvt_double;
|
|
break;
|
|
|
|
case op_product:
|
|
p.full_values[i].s = milo::dtoa_milo(atof(p.full_values[i].s.c_str()) * atof(val.s.c_str()));
|
|
p.full_values[i].type = mvt_double;
|
|
break;
|
|
|
|
case op_max: {
|
|
double existing = atof(p.full_values[i].s.c_str());
|
|
double maybe = atof(val.s.c_str());
|
|
if (maybe > existing) {
|
|
p.full_values[i].s = val.s.c_str();
|
|
p.full_values[i].type = mvt_double;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case op_min: {
|
|
double existing = atof(p.full_values[i].s.c_str());
|
|
double maybe = atof(val.s.c_str());
|
|
if (maybe < existing) {
|
|
p.full_values[i].s = val.s.c_str();
|
|
p.full_values[i].type = mvt_double;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case op_mean: {
|
|
auto state = p.attribute_accum_state.find(key);
|
|
if (state == p.attribute_accum_state.end()) {
|
|
accum_state s;
|
|
s.sum = atof(p.full_values[i].s.c_str()) + atof(val.s.c_str());
|
|
s.count = 2;
|
|
p.attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));
|
|
|
|
p.full_values[i].s = milo::dtoa_milo(s.sum / s.count);
|
|
} else {
|
|
state->second.sum += atof(val.s.c_str());
|
|
state->second.count += 1;
|
|
|
|
p.full_values[i].s = milo::dtoa_milo(state->second.sum / state->second.count);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case op_concat:
|
|
p.full_values[i].s += val.s;
|
|
p.full_values[i].type = mvt_string;
|
|
break;
|
|
|
|
case op_comma:
|
|
p.full_values[i].s += std::string(",") + val.s;
|
|
p.full_values[i].type = mvt_string;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void preserve_attributes(std::map<std::string, attribute_op> const *attribute_accum, serial_feature &sf, char *stringpool, long long *pool_off, partial &p) {
|
|
for (size_t i = 0; i < sf.keys.size(); i++) {
|
|
std::string key = stringpool + pool_off[sf.segment] + sf.keys[i] + 1;
|
|
|
|
serial_val sv;
|
|
sv.type = (stringpool + pool_off[sf.segment])[sf.values[i]];
|
|
sv.s = stringpool + pool_off[sf.segment] + sf.values[i] + 1;
|
|
|
|
auto f = attribute_accum->find(key);
|
|
if (f != attribute_accum->end()) {
|
|
preserve_attribute(f->second, sf, stringpool, pool_off, key, sv, p);
|
|
}
|
|
}
|
|
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
|
std::string key = sf.full_keys[i];
|
|
serial_val sv = sf.full_values[i];
|
|
|
|
auto f = attribute_accum->find(key);
|
|
if (f != attribute_accum->end()) {
|
|
preserve_attribute(f->second, sf, stringpool, pool_off, key, sv, p);
|
|
}
|
|
}
|
|
}
|
|
|
|
bool find_partial(std::vector<partial> &partials, serial_feature &sf, ssize_t &out, std::vector<std::vector<std::string>> *layer_unmaps, long long maxextent) {
|
|
for (size_t i = partials.size(); i > 0; i--) {
|
|
if (partials[i - 1].t == sf.t) {
|
|
std::string &layername1 = (*layer_unmaps)[partials[i - 1].segment][partials[i - 1].layer];
|
|
std::string &layername2 = (*layer_unmaps)[sf.segment][sf.layer];
|
|
|
|
if (layername1 == layername2 && partials[i - 1].extent <= maxextent) {
|
|
out = i - 1;
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
|
|
if (geometry.size() == 0) {
|
|
return true;
|
|
}
|
|
|
|
long long x = geometry[0].x >> (32 - detail - z);
|
|
long long y = geometry[0].y >> (32 - detail - z);
|
|
|
|
for (auto &g : geometry) {
|
|
long long xx = g.x >> (32 - detail - z);
|
|
long long yy = g.y >> (32 - detail - z);
|
|
|
|
if (xx != x || yy != y) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Keep only a sample of 100K extents for feature dropping,
|
|
// to avoid spending lots of memory on a complete list when there are
|
|
// hundreds of millions of features.
|
|
template <class T>
|
|
void add_sample_to(std::vector<T> &vals, T val, size_t &increment, size_t seq) {
|
|
if (seq % increment == 0) {
|
|
vals.push_back(val);
|
|
|
|
if (vals.size() > 100000) {
|
|
std::vector<T> tmp;
|
|
|
|
for (size_t i = 0; i < vals.size(); i += 2) {
|
|
tmp.push_back(vals[i]);
|
|
}
|
|
|
|
increment *= 2;
|
|
vals = tmp;
|
|
}
|
|
}
|
|
}
|
|
|
|
long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg, atomic_strategy *strategy) {
|
|
double merge_fraction = 1;
|
|
double mingap_fraction = 1;
|
|
double minextent_fraction = 1;
|
|
|
|
static std::atomic<double> oprogress(0);
|
|
long long og = *geompos_in;
|
|
|
|
// XXX is there a way to do this without floating point?
|
|
int max_zoom_increment = std::log(child_shards) / std::log(4);
|
|
if (child_shards < 4 || max_zoom_increment < 1) {
|
|
fprintf(stderr, "Internal error: %d shards, max zoom increment %d\n", child_shards, max_zoom_increment);
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
if ((((child_shards - 1) << 1) & child_shards) != child_shards) {
|
|
fprintf(stderr, "Internal error: %d shards not a power of 2\n", child_shards);
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
|
|
int nextzoom = z + 1;
|
|
if (nextzoom < minzoom) {
|
|
if (z + max_zoom_increment > minzoom) {
|
|
nextzoom = minzoom;
|
|
} else {
|
|
nextzoom = z + max_zoom_increment;
|
|
}
|
|
}
|
|
|
|
bool first_time = true;
|
|
// This only loops if the tile data didn't fit, in which case the detail
|
|
// goes down and the progress indicator goes backward for the next try.
|
|
int line_detail;
|
|
for (line_detail = detail; line_detail >= min_detail || line_detail == detail; line_detail--, oprogress = 0) {
|
|
long long count = 0;
|
|
double accum_area = 0;
|
|
|
|
double fraction_accum = 0;
|
|
|
|
unsigned long long previndex = 0, density_previndex = 0, merge_previndex = 0;
|
|
unsigned long long extent_previndex = 0;
|
|
double scale = (double) (1LL << (64 - 2 * (z + 8)));
|
|
double gap = 0, density_gap = 0;
|
|
double spacing = 0;
|
|
|
|
long long original_features = 0;
|
|
long long unclipped_features = 0;
|
|
|
|
std::vector<struct partial> partials;
|
|
std::map<std::string, std::vector<coalesce>> layers;
|
|
|
|
std::vector<unsigned long long> indices;
|
|
std::vector<long long> extents;
|
|
size_t extents_increment = 1;
|
|
|
|
double coalesced_area = 0;
|
|
drawvec shared_nodes;
|
|
|
|
int tile_detail = line_detail;
|
|
size_t skipped = 0;
|
|
size_t kept = 0;
|
|
|
|
size_t unsimplified_geometry_size = 0;
|
|
size_t simplified_geometry_through = 0;
|
|
|
|
int within[child_shards];
|
|
std::atomic<long long> geompos[child_shards];
|
|
for (size_t i = 0; i < (size_t) child_shards; i++) {
|
|
geompos[i] = 0;
|
|
within[i] = 0;
|
|
}
|
|
|
|
if (*geompos_in != og) {
|
|
if (fseek(geoms, og, SEEK_SET) != 0) {
|
|
perror("fseek geom");
|
|
exit(EXIT_SEEK);
|
|
}
|
|
*geompos_in = og;
|
|
}
|
|
|
|
int prefilter_write = -1, prefilter_read = -1;
|
|
pid_t prefilter_pid = 0;
|
|
FILE *prefilter_fp = NULL;
|
|
pthread_t prefilter_writer;
|
|
run_prefilter_args rpa; // here so it stays in scope until joined
|
|
FILE *prefilter_read_fp = NULL;
|
|
json_pull *prefilter_jp = NULL;
|
|
|
|
serial_feature tiny_feature;
|
|
|
|
if (z < minzoom) {
|
|
prefilter = NULL;
|
|
postfilter = NULL;
|
|
}
|
|
|
|
if (prefilter != NULL) {
|
|
setup_filter(prefilter, &prefilter_write, &prefilter_read, &prefilter_pid, z, tx, ty);
|
|
prefilter_fp = fdopen(prefilter_write, "w");
|
|
if (prefilter_fp == NULL) {
|
|
perror("freopen prefilter");
|
|
exit(EXIT_OPEN);
|
|
}
|
|
|
|
rpa.geoms = geoms;
|
|
rpa.geompos_in = geompos_in;
|
|
rpa.z = z;
|
|
rpa.tx = tx;
|
|
rpa.ty = ty;
|
|
rpa.initial_x = initial_x;
|
|
rpa.initial_y = initial_y;
|
|
rpa.original_features = &original_features;
|
|
rpa.unclipped_features = &unclipped_features;
|
|
rpa.nextzoom = nextzoom;
|
|
rpa.maxzoom = maxzoom;
|
|
rpa.minzoom = minzoom;
|
|
rpa.max_zoom_increment = max_zoom_increment;
|
|
rpa.pass = pass;
|
|
rpa.along = along;
|
|
rpa.alongminus = alongminus;
|
|
rpa.buffer = buffer;
|
|
rpa.within = within;
|
|
rpa.geomfile = geomfile;
|
|
rpa.geompos = geompos;
|
|
rpa.oprogress = &oprogress;
|
|
rpa.todo = todo;
|
|
rpa.fname = fname;
|
|
rpa.child_shards = child_shards;
|
|
rpa.prefilter_fp = prefilter_fp;
|
|
rpa.layer_unmaps = layer_unmaps;
|
|
rpa.stringpool = stringpool;
|
|
rpa.pool_off = pool_off;
|
|
rpa.filter = filter;
|
|
rpa.first_time = first_time;
|
|
|
|
if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
|
|
perror("pthread_create (prefilter writer)");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
|
|
prefilter_read_fp = fdopen(prefilter_read, "r");
|
|
if (prefilter_read_fp == NULL) {
|
|
perror("fdopen prefilter output");
|
|
exit(EXIT_OPEN);
|
|
}
|
|
prefilter_jp = json_begin_file(prefilter_read_fp);
|
|
}
|
|
|
|
for (size_t seq = 0;; seq++) {
|
|
serial_feature sf;
|
|
ssize_t which_partial = -1;
|
|
|
|
if (prefilter == NULL) {
|
|
sf = next_feature(geoms, geompos_in, z, tx, ty, initial_x, initial_y, &original_features, &unclipped_features, nextzoom, maxzoom, minzoom, max_zoom_increment, pass, along, alongminus, buffer, within, geomfile, geompos, &oprogress, todo, fname, child_shards, filter, stringpool, pool_off, layer_unmaps, first_time);
|
|
} else {
|
|
sf = parse_feature(prefilter_jp, z, tx, ty, layermaps, tiling_seg, layer_unmaps, postfilter != NULL);
|
|
}
|
|
|
|
if (sf.t < 0) {
|
|
break;
|
|
}
|
|
|
|
if (sf.t == VT_POINT) {
|
|
if (extent_previndex >= sf.index) {
|
|
sf.extent = 1;
|
|
} else {
|
|
double radius = sqrt(sf.index - extent_previndex) / 4.0;
|
|
sf.extent = M_PI * radius * radius;
|
|
if (sf.extent < 1) {
|
|
sf.extent = 1;
|
|
}
|
|
}
|
|
|
|
extent_previndex = sf.index;
|
|
}
|
|
|
|
if (sf.dropped) {
|
|
if (find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->dropped_by_rate++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (gamma > 0) {
|
|
if (manage_gap(sf.index, &previndex, scale, gamma, &gap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->dropped_by_gamma++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Cap the indices, rather than sampling them like extents (areas),
|
|
// because choose_mingap cares about the distance between *surviving*
|
|
// features, not between *original* features, so we can't just store
|
|
// gaps rather than indices to be able to downsample them fairly.
|
|
// Hopefully the first 100K features in the tile are reasonably
|
|
// representative of the other features in the tile.
|
|
const size_t MAX_INDICES = 100000;
|
|
|
|
if (additional[A_CLUSTER_DENSEST_AS_NEEDED] || cluster_distance != 0) {
|
|
if (indices.size() < MAX_INDICES) {
|
|
indices.push_back(sf.index);
|
|
}
|
|
if ((sf.index < merge_previndex || sf.index - merge_previndex < mingap) && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
partials[which_partial].clustered++;
|
|
|
|
if (partials[which_partial].t == VT_POINT &&
|
|
partials[which_partial].geoms.size() == 1 &&
|
|
partials[which_partial].geoms[0].size() == 1 &&
|
|
sf.geometry.size() == 1) {
|
|
double x = (double) partials[which_partial].geoms[0][0].x * partials[which_partial].clustered;
|
|
double y = (double) partials[which_partial].geoms[0][0].y * partials[which_partial].clustered;
|
|
x += sf.geometry[0].x;
|
|
y += sf.geometry[0].y;
|
|
partials[which_partial].geoms[0][0].x = x / (partials[which_partial].clustered + 1);
|
|
partials[which_partial].geoms[0][0].y = y / (partials[which_partial].clustered + 1);
|
|
}
|
|
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->coalesced_as_needed++;
|
|
continue;
|
|
}
|
|
} else if (additional[A_DROP_DENSEST_AS_NEEDED]) {
|
|
if (indices.size() < MAX_INDICES) {
|
|
indices.push_back(sf.index);
|
|
}
|
|
if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->dropped_as_needed++;
|
|
continue;
|
|
}
|
|
} else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) {
|
|
if (indices.size() < MAX_INDICES) {
|
|
indices.push_back(sf.index);
|
|
}
|
|
if (sf.index - merge_previndex < mingap && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
partials[which_partial].geoms.push_back(sf.geometry);
|
|
partials[which_partial].coalesced = true;
|
|
coalesced_area += sf.extent;
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->coalesced_as_needed++;
|
|
continue;
|
|
}
|
|
} else if (additional[A_DROP_SMALLEST_AS_NEEDED]) {
|
|
add_sample_to(extents, sf.extent, extents_increment, seq);
|
|
if (sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, minextent)) {
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->dropped_as_needed++;
|
|
continue;
|
|
}
|
|
} else if (additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
|
|
add_sample_to(extents, sf.extent, extents_increment, seq);
|
|
if (sf.extent + coalesced_area <= minextent && find_partial(partials, sf, which_partial, layer_unmaps, minextent)) {
|
|
partials[which_partial].geoms.push_back(sf.geometry);
|
|
partials[which_partial].coalesced = true;
|
|
coalesced_area += sf.extent;
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
strategy->coalesced_as_needed++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (additional[A_CALCULATE_FEATURE_DENSITY]) {
|
|
// Gamma is always 1 for this calculation so there is a reasonable
|
|
// interpretation when no features are being dropped.
|
|
// The spacing is only calculated if a feature would be retained by
|
|
// that standard, so that duplicates aren't reported as infinitely dense.
|
|
|
|
double o_density_previndex = density_previndex;
|
|
if (!manage_gap(sf.index, &density_previndex, scale, 1, &density_gap)) {
|
|
spacing = (sf.index - o_density_previndex) / scale;
|
|
}
|
|
}
|
|
|
|
fraction_accum += fraction;
|
|
if (fraction_accum < 1 && find_partial(partials, sf, which_partial, layer_unmaps, LLONG_MAX)) {
|
|
if (additional[A_COALESCE_FRACTION_AS_NEEDED]) {
|
|
partials[which_partial].geoms.push_back(sf.geometry);
|
|
partials[which_partial].coalesced = true;
|
|
coalesced_area += sf.extent;
|
|
strategy->coalesced_as_needed++;
|
|
} else {
|
|
strategy->dropped_as_needed++;
|
|
}
|
|
preserve_attributes(arg->attribute_accum, sf, stringpool, pool_off, partials[which_partial]);
|
|
continue;
|
|
}
|
|
fraction_accum -= 1;
|
|
|
|
bool reduced = false;
|
|
if (sf.t == VT_POLYGON) {
|
|
if (!prevent[P_TINY_POLYGON_REDUCTION] && !additional[A_GRID_LOW_ZOOMS]) {
|
|
sf.geometry = reduce_tiny_poly(sf.geometry, z, line_detail, &reduced, &accum_area, &sf, &tiny_feature);
|
|
if (reduced) {
|
|
strategy->tiny_polygons++;
|
|
}
|
|
if (sf.geometry.size() == 0) {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
if (sf.t == VT_POLYGON || sf.t == VT_LINE) {
|
|
if (line_is_too_small(sf.geometry, z, line_detail)) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (sf.geometry.size() > 0) {
|
|
if (partials.size() > max_tile_size) {
|
|
// Even being maximally conservative, each feature is still going to be
|
|
// at least one byte in the output tile, so this can't possibly work.
|
|
skipped++;
|
|
} else {
|
|
kept++;
|
|
|
|
if (prevent[P_SIMPLIFY_SHARED_NODES]) {
|
|
for (auto &g : sf.geometry) {
|
|
shared_nodes.push_back(g);
|
|
}
|
|
}
|
|
|
|
partial p;
|
|
p.geoms.push_back(sf.geometry);
|
|
p.layer = sf.layer;
|
|
p.t = sf.t;
|
|
p.segment = sf.segment;
|
|
p.original_seq = sf.seq;
|
|
p.reduced = reduced;
|
|
p.coalesced = false;
|
|
p.z = z;
|
|
p.tx = tx;
|
|
p.ty = ty;
|
|
p.line_detail = line_detail;
|
|
p.extra_detail = line_detail;
|
|
p.maxzoom = maxzoom;
|
|
p.keys = sf.keys;
|
|
p.values = sf.values;
|
|
p.full_keys = sf.full_keys;
|
|
p.full_values = sf.full_values;
|
|
p.spacing = spacing;
|
|
p.simplification = simplification;
|
|
p.id = sf.id;
|
|
p.has_id = sf.has_id;
|
|
p.index = sf.index;
|
|
p.label_point = sf.label_point;
|
|
p.renamed = -1;
|
|
p.extent = sf.extent;
|
|
p.clustered = 0;
|
|
|
|
if (line_detail == detail && extra_detail >= 0 && z == maxzoom) {
|
|
p.extra_detail = extra_detail;
|
|
// maximum allowed coordinate delta in geometries is 2^31 - 1
|
|
// so we need to stay under that, including the buffer
|
|
if (p.extra_detail >= 30 - z) {
|
|
p.extra_detail = 30 - z;
|
|
}
|
|
tile_detail = p.extra_detail;
|
|
}
|
|
|
|
partials.push_back(p);
|
|
|
|
unsimplified_geometry_size += sf.geometry.size() * sizeof(draw);
|
|
if (unsimplified_geometry_size > 10 * 1024 * 1024 && !additional[A_DETECT_SHARED_BORDERS]) {
|
|
drawvec dv;
|
|
|
|
for (; simplified_geometry_through < partials.size(); simplified_geometry_through++) {
|
|
simplify_partial(&partials[simplified_geometry_through], dv);
|
|
|
|
for (auto &g : partials[simplified_geometry_through].geoms) {
|
|
if (partials[simplified_geometry_through].t == VT_POLYGON) {
|
|
g = clean_or_clip_poly(g, 0, 0, false);
|
|
}
|
|
}
|
|
}
|
|
|
|
unsimplified_geometry_size = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
merge_previndex = sf.index;
|
|
coalesced_area = 0;
|
|
}
|
|
|
|
{
|
|
drawvec just_shared_nodes;
|
|
std::sort(shared_nodes.begin(), shared_nodes.end());
|
|
|
|
for (size_t i = 0; i + 1 < shared_nodes.size(); i++) {
|
|
if (shared_nodes[i] == shared_nodes[i + 1]) {
|
|
just_shared_nodes.push_back(shared_nodes[i]);
|
|
|
|
draw d = shared_nodes[i];
|
|
i++;
|
|
while (i + 1 < shared_nodes.size() && shared_nodes[i + 1] == d) {
|
|
i++;
|
|
}
|
|
}
|
|
}
|
|
|
|
shared_nodes = just_shared_nodes;
|
|
}
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
partial &p = partials[i];
|
|
|
|
if (p.clustered > 0) {
|
|
std::string layername = (*layer_unmaps)[p.segment][p.layer];
|
|
serial_val sv, sv2, sv3;
|
|
|
|
p.full_keys.push_back("clustered");
|
|
sv.type = mvt_bool;
|
|
sv.s = "true";
|
|
p.full_values.push_back(sv);
|
|
|
|
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "clustered", sv);
|
|
|
|
p.full_keys.push_back("point_count");
|
|
sv2.type = mvt_double;
|
|
sv2.s = std::to_string(p.clustered + 1);
|
|
p.full_values.push_back(sv2);
|
|
|
|
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "point_count", sv2);
|
|
|
|
p.full_keys.push_back("sqrt_point_count");
|
|
sv3.type = mvt_double;
|
|
sv3.s = std::to_string(round(100 * sqrt(p.clustered + 1)) / 100.0);
|
|
p.full_values.push_back(sv3);
|
|
|
|
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, "sqrt_point_count", sv3);
|
|
}
|
|
|
|
if (p.need_tilestats.size() > 0) {
|
|
std::string layername = (*layer_unmaps)[p.segment][p.layer];
|
|
|
|
for (size_t j = 0; j < p.full_keys.size(); j++) {
|
|
if (p.need_tilestats.count(p.full_keys[j]) > 0) {
|
|
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, p.full_keys[j], p.full_values[j]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (prefilter != NULL) {
|
|
json_end(prefilter_jp);
|
|
if (fclose(prefilter_read_fp) != 0) {
|
|
perror("close output from prefilter");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
while (1) {
|
|
int stat_loc;
|
|
if (waitpid(prefilter_pid, &stat_loc, 0) < 0) {
|
|
perror("waitpid for prefilter\n");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
if (WIFEXITED(stat_loc) || WIFSIGNALED(stat_loc)) {
|
|
break;
|
|
}
|
|
}
|
|
void *ret;
|
|
if (pthread_join(prefilter_writer, &ret) != 0) {
|
|
perror("pthread_join prefilter writer");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
}
|
|
|
|
first_time = false;
|
|
|
|
if (additional[A_DETECT_SHARED_BORDERS]) {
|
|
find_common_edges(partials, z, line_detail, simplification, maxzoom, merge_fraction);
|
|
}
|
|
|
|
int tasks = ceil((double) CPUS / *running);
|
|
if (tasks < 1) {
|
|
tasks = 1;
|
|
}
|
|
|
|
pthread_t pthreads[tasks];
|
|
std::vector<partial_arg> args;
|
|
args.resize(tasks);
|
|
for (int i = 0; i < tasks; i++) {
|
|
args[i].task = i;
|
|
args[i].tasks = tasks;
|
|
args[i].partials = &partials;
|
|
args[i].shared_nodes = &shared_nodes;
|
|
|
|
if (tasks > 1) {
|
|
if (pthread_create(&pthreads[i], NULL, partial_feature_worker, &args[i]) != 0) {
|
|
perror("pthread_create");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
} else {
|
|
partial_feature_worker(&args[i]);
|
|
}
|
|
}
|
|
|
|
if (tasks > 1) {
|
|
for (int i = 0; i < tasks; i++) {
|
|
void *retval;
|
|
|
|
if (pthread_join(pthreads[i], &retval) != 0) {
|
|
perror("pthread_join");
|
|
}
|
|
}
|
|
}
|
|
|
|
for (size_t i = 0; i < partials.size(); i++) {
|
|
std::vector<drawvec> &pgeoms = partials[i].geoms;
|
|
signed char t = partials[i].t;
|
|
long long original_seq = partials[i].original_seq;
|
|
|
|
// A complex polygon may have been split up into multiple geometries.
|
|
// Break them out into multiple features if necessary.
|
|
for (size_t j = 0; j < pgeoms.size(); j++) {
|
|
if (t == VT_POINT || draws_something(pgeoms[j])) {
|
|
struct coalesce c;
|
|
|
|
c.type = t;
|
|
c.index = partials[i].index;
|
|
c.geom = pgeoms[j];
|
|
pgeoms[j].clear();
|
|
c.coalesced = false;
|
|
c.original_seq = original_seq;
|
|
c.stringpool = stringpool + pool_off[partials[i].segment];
|
|
c.keys = partials[i].keys;
|
|
c.values = partials[i].values;
|
|
c.full_keys = partials[i].full_keys;
|
|
c.full_values = partials[i].full_values;
|
|
c.spacing = partials[i].spacing;
|
|
c.id = partials[i].id;
|
|
c.has_id = partials[i].has_id;
|
|
c.extent = partials[i].extent;
|
|
|
|
// printf("segment %d layer %lld is %s\n", partials[i].segment, partials[i].layer, (*layer_unmaps)[partials[i].segment][partials[i].layer].c_str());
|
|
|
|
std::string layername = (*layer_unmaps)[partials[i].segment][partials[i].layer];
|
|
if (layers.count(layername) == 0) {
|
|
layers.insert(std::pair<std::string, std::vector<coalesce>>(layername, std::vector<coalesce>()));
|
|
}
|
|
|
|
auto l = layers.find(layername);
|
|
if (l == layers.end()) {
|
|
fprintf(stderr, "Internal error: couldn't find layer %s\n", layername.c_str());
|
|
fprintf(stderr, "segment %d\n", partials[i].segment);
|
|
fprintf(stderr, "layer %lld\n", partials[i].layer);
|
|
exit(EXIT_IMPOSSIBLE);
|
|
}
|
|
l->second.push_back(c);
|
|
}
|
|
}
|
|
}
|
|
|
|
partials.clear();
|
|
|
|
int j;
|
|
for (j = 0; j < child_shards; j++) {
|
|
if (within[j]) {
|
|
serialize_byte(geomfile[j], -2, &geompos[j], fname);
|
|
within[j] = 0;
|
|
}
|
|
}
|
|
|
|
for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
|
|
std::vector<coalesce> &layer_features = layer_iterator->second;
|
|
|
|
if (additional[A_REORDER]) {
|
|
std::sort(layer_features.begin(), layer_features.end());
|
|
}
|
|
|
|
std::vector<coalesce> out;
|
|
if (layer_features.size() > 0) {
|
|
out.push_back(layer_features[0]);
|
|
}
|
|
for (size_t x = 1; x < layer_features.size(); x++) {
|
|
size_t y = out.size() - 1;
|
|
|
|
#if 0
|
|
if (out.size() > 0 && coalcmp(&layer_features[x], &out[y]) < 0) {
|
|
fprintf(stderr, "\nfeature out of order\n");
|
|
}
|
|
#endif
|
|
|
|
if (additional[A_COALESCE] && out.size() > 0 && coalcmp(&layer_features[x], &out[y]) == 0) {
|
|
for (size_t g = 0; g < layer_features[x].geom.size(); g++) {
|
|
out[y].geom.push_back(layer_features[x].geom[g]);
|
|
}
|
|
out[y].coalesced = true;
|
|
} else {
|
|
out.push_back(layer_features[x]);
|
|
}
|
|
}
|
|
|
|
layer_features = out;
|
|
|
|
out.clear();
|
|
for (size_t x = 0; x < layer_features.size(); x++) {
|
|
if (layer_features[x].coalesced && layer_features[x].type == VT_LINE) {
|
|
layer_features[x].geom = remove_noop(layer_features[x].geom, layer_features[x].type, 0);
|
|
layer_features[x].geom = simplify_lines(layer_features[x].geom, 32, 0,
|
|
!(prevent[P_CLIPPING] || prevent[P_DUPLICATION]), simplification, layer_features[x].type == VT_POLYGON ? 4 : 0, shared_nodes);
|
|
}
|
|
|
|
if (layer_features[x].type == VT_POLYGON) {
|
|
if (layer_features[x].coalesced) {
|
|
layer_features[x].geom = clean_or_clip_poly(layer_features[x].geom, 0, 0, false);
|
|
}
|
|
|
|
layer_features[x].geom = close_poly(layer_features[x].geom);
|
|
}
|
|
|
|
if (layer_features[x].geom.size() > 0) {
|
|
out.push_back(layer_features[x]);
|
|
}
|
|
}
|
|
layer_features = out;
|
|
|
|
if (prevent[P_INPUT_ORDER]) {
|
|
std::sort(layer_features.begin(), layer_features.end(), preservecmp);
|
|
}
|
|
|
|
if (order_by.size() != 0) {
|
|
std::sort(layer_features.begin(), layer_features.end(), ordercmp);
|
|
}
|
|
|
|
if (z == maxzoom && limit_tile_feature_count_at_maxzoom != 0) {
|
|
if (layer_features.size() > limit_tile_feature_count_at_maxzoom) {
|
|
layer_features.resize(limit_tile_feature_count_at_maxzoom);
|
|
}
|
|
} else if (limit_tile_feature_count != 0) {
|
|
if (layer_features.size() > limit_tile_feature_count) {
|
|
layer_features.resize(limit_tile_feature_count);
|
|
}
|
|
}
|
|
}
|
|
|
|
mvt_tile tile;
|
|
|
|
for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
|
|
std::vector<coalesce> &layer_features = layer_iterator->second;
|
|
|
|
mvt_layer layer;
|
|
layer.name = layer_iterator->first;
|
|
layer.version = 2;
|
|
layer.extent = 1 << tile_detail;
|
|
|
|
for (size_t x = 0; x < layer_features.size(); x++) {
|
|
mvt_feature feature;
|
|
|
|
if (layer_features[x].type == VT_LINE || layer_features[x].type == VT_POLYGON) {
|
|
layer_features[x].geom = remove_noop(layer_features[x].geom, layer_features[x].type, 0);
|
|
}
|
|
|
|
if (layer_features[x].geom.size() == 0) {
|
|
continue;
|
|
}
|
|
|
|
feature.type = layer_features[x].type;
|
|
feature.geometry = to_feature(layer_features[x].geom);
|
|
count += layer_features[x].geom.size();
|
|
layer_features[x].geom.clear();
|
|
|
|
feature.id = layer_features[x].id;
|
|
feature.has_id = layer_features[x].has_id;
|
|
|
|
decode_meta(layer_features[x].keys, layer_features[x].values, layer_features[x].stringpool, layer, feature);
|
|
for (size_t a = 0; a < layer_features[x].full_keys.size(); a++) {
|
|
serial_val sv = layer_features[x].full_values[a];
|
|
mvt_value v = stringified_to_mvt_value(sv.type, sv.s.c_str());
|
|
layer.tag(feature, layer_features[x].full_keys[a], v);
|
|
}
|
|
|
|
if (additional[A_CALCULATE_FEATURE_DENSITY]) {
|
|
int glow = 255;
|
|
if (layer_features[x].spacing > 0) {
|
|
glow = (1 / layer_features[x].spacing);
|
|
if (glow > 255) {
|
|
glow = 255;
|
|
}
|
|
}
|
|
|
|
mvt_value v;
|
|
v.type = mvt_sint;
|
|
v.numeric_value.sint_value = glow;
|
|
layer.tag(feature, "tippecanoe_feature_density", v);
|
|
|
|
serial_val sv;
|
|
sv.type = mvt_double;
|
|
sv.s = std::to_string(glow);
|
|
|
|
add_tilestats(layer.name, z, layermaps, tiling_seg, layer_unmaps, "tippecanoe_feature_density", sv);
|
|
}
|
|
|
|
layer.features.push_back(feature);
|
|
}
|
|
|
|
if (layer.features.size() > 0) {
|
|
tile.layers.push_back(layer);
|
|
}
|
|
}
|
|
|
|
if (postfilter != NULL) {
|
|
tile.layers = filter_layers(postfilter, tile.layers, z, tx, ty, layermaps, tiling_seg, layer_unmaps, 1 << tile_detail);
|
|
}
|
|
|
|
if (z == 0 && unclipped_features < original_features / 2 && clipbboxes.size() == 0) {
|
|
fprintf(stderr, "\n\nMore than half the features were clipped away at zoom level 0.\n");
|
|
fprintf(stderr, "Is your data in the wrong projection? It should be in WGS84/EPSG:4326.\n");
|
|
}
|
|
|
|
size_t totalsize = 0;
|
|
for (auto layer_iterator = layers.begin(); layer_iterator != layers.end(); ++layer_iterator) {
|
|
std::vector<coalesce> &layer_features = layer_iterator->second;
|
|
totalsize += layer_features.size();
|
|
}
|
|
|
|
size_t passes = pass + 1;
|
|
double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + pass) / passes + z) / (maxzoom + 1) * 1000) / 10;
|
|
if (progress >= oprogress + 0.1) {
|
|
if (!quiet && !quiet_progress && progress_time()) {
|
|
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty);
|
|
}
|
|
if (logger.json_enabled && progress_time()) {
|
|
logger.progress_tile(progress);
|
|
}
|
|
oprogress = progress;
|
|
}
|
|
|
|
if (totalsize > 0 && tile.layers.size() > 0) {
|
|
if (totalsize > max_tile_features && !prevent[P_FEATURE_LIMIT]) {
|
|
if (totalsize > arg->feature_count_out) {
|
|
arg->feature_count_out = totalsize;
|
|
}
|
|
|
|
if (!quiet) {
|
|
fprintf(stderr, "tile %d/%u/%u has %zu features, >%zu \n", z, tx, ty, totalsize, max_tile_features);
|
|
}
|
|
|
|
if (additional[A_INCREASE_GAMMA_AS_NEEDED] && gamma < 10) {
|
|
if (gamma < 1) {
|
|
gamma = 1;
|
|
} else {
|
|
gamma = gamma * 1.25;
|
|
}
|
|
|
|
if (gamma > arg->gamma_out) {
|
|
arg->gamma_out = gamma;
|
|
arg->still_dropping = true;
|
|
}
|
|
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try gamma of %0.3f to make it fit\n", gamma);
|
|
}
|
|
line_detail++; // to keep it the same when the loop decrements it
|
|
continue;
|
|
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
|
|
mingap_fraction = mingap_fraction * max_tile_features / totalsize * 0.90;
|
|
unsigned long long mg = choose_mingap(indices, mingap_fraction);
|
|
if (mg <= mingap) {
|
|
mg = (mingap + 1) * 1.5;
|
|
|
|
if (mg <= mingap) {
|
|
mg = ULONG_MAX;
|
|
}
|
|
}
|
|
mingap = mg;
|
|
if (mingap > arg->mingap_out) {
|
|
arg->mingap_out = mingap;
|
|
arg->still_dropping = true;
|
|
}
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping the sparsest %0.2f%% of the features to make it fit\n", mingap_fraction * 100.0);
|
|
}
|
|
line_detail++;
|
|
continue;
|
|
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
|
|
minextent_fraction = minextent_fraction * max_tile_features / totalsize * 0.75;
|
|
long long m = choose_minextent(extents, minextent_fraction);
|
|
if (m != minextent) {
|
|
minextent = m;
|
|
if (minextent > arg->minextent_out) {
|
|
arg->minextent_out = minextent;
|
|
arg->still_dropping = true;
|
|
}
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping the biggest %0.2f%% of the features to make it fit\n", minextent_fraction * 100.0);
|
|
}
|
|
line_detail++;
|
|
continue;
|
|
}
|
|
} else if (totalsize > layers.size() && (prevent[P_DYNAMIC_DROP] || additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED])) {
|
|
// The 95% is a guess to avoid too many retries
|
|
// and probably actually varies based on how much duplicated metadata there is
|
|
|
|
fraction = fraction * max_tile_features / totalsize * 0.95;
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
|
|
}
|
|
if ((additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED]) && fraction < arg->fraction_out) {
|
|
arg->fraction_out = fraction;
|
|
arg->still_dropping = true;
|
|
} else if (prevent[P_DYNAMIC_DROP]) {
|
|
arg->still_dropping = true;
|
|
}
|
|
line_detail++; // to keep it the same when the loop decrements it
|
|
continue;
|
|
} else {
|
|
fprintf(stderr, "Try using --drop-fraction-as-needed or --drop-densest-as-needed.\n");
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
std::string compressed;
|
|
std::string pbf = tile.encode();
|
|
|
|
if (!prevent[P_TILE_COMPRESSION]) {
|
|
compress(pbf, compressed);
|
|
} else {
|
|
compressed = pbf;
|
|
}
|
|
|
|
if (compressed.size() > max_tile_size && !prevent[P_KILOBYTE_LIMIT]) {
|
|
// Estimate how big it really should have been compressed
|
|
// from how many features were kept vs skipped for already being
|
|
// over the threshold
|
|
|
|
double kept_adjust = (skipped + kept) / (double) kept;
|
|
|
|
if (compressed.size() > arg->tile_size_out) {
|
|
arg->tile_size_out = compressed.size() * kept_adjust;
|
|
}
|
|
|
|
if (!quiet) {
|
|
if (skipped > 0) {
|
|
fprintf(stderr, "tile %d/%u/%u size is %lld (probably really %lld) with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), (long long) (compressed.size() * kept_adjust), line_detail, max_tile_size);
|
|
} else {
|
|
fprintf(stderr, "tile %d/%u/%u size is %lld with detail %d, >%zu \n", z, tx, ty, (long long) compressed.size(), line_detail, max_tile_size);
|
|
}
|
|
}
|
|
|
|
if (additional[A_INCREASE_GAMMA_AS_NEEDED] && gamma < 10) {
|
|
if (gamma < 1) {
|
|
gamma = 1;
|
|
} else {
|
|
gamma = gamma * 1.25;
|
|
}
|
|
|
|
if (gamma > arg->gamma_out) {
|
|
arg->gamma_out = gamma;
|
|
arg->still_dropping = true;
|
|
}
|
|
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try gamma of %0.3f to make it fit\n", gamma);
|
|
}
|
|
line_detail++; // to keep it the same when the loop decrements it
|
|
} else if (mingap < ULONG_MAX && (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED])) {
|
|
mingap_fraction = mingap_fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.90;
|
|
unsigned long long mg = choose_mingap(indices, mingap_fraction);
|
|
if (mg <= mingap) {
|
|
double nmg = (mingap + 1) * 1.5;
|
|
|
|
if (nmg <= mingap || nmg > ULONG_MAX) {
|
|
mg = ULONG_MAX;
|
|
} else {
|
|
mg = nmg;
|
|
|
|
if (mg <= mingap) {
|
|
mg = ULONG_MAX;
|
|
}
|
|
}
|
|
}
|
|
mingap = mg;
|
|
if (mingap > arg->mingap_out) {
|
|
arg->mingap_out = mingap;
|
|
arg->still_dropping = true;
|
|
}
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping the sparsest %0.2f%% of the features to make it fit\n", mingap_fraction * 100.0);
|
|
}
|
|
line_detail++;
|
|
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
|
|
minextent_fraction = minextent_fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.75;
|
|
long long m = choose_minextent(extents, minextent_fraction);
|
|
if (m != minextent) {
|
|
minextent = m;
|
|
if (minextent > arg->minextent_out) {
|
|
arg->minextent_out = minextent;
|
|
arg->still_dropping = true;
|
|
}
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping the biggest %0.2f%% of the features to make it fit\n", minextent_fraction * 100.0);
|
|
}
|
|
line_detail++;
|
|
continue;
|
|
}
|
|
} else if (totalsize > layers.size() && (prevent[P_DYNAMIC_DROP] || additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED])) {
|
|
// The 95% is a guess to avoid too many retries
|
|
// and probably actually varies based on how much duplicated metadata there is
|
|
|
|
fraction = fraction * max_tile_size / (kept_adjust * compressed.size()) * 0.95;
|
|
if (!quiet) {
|
|
fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
|
|
}
|
|
if ((additional[A_DROP_FRACTION_AS_NEEDED] || additional[A_COALESCE_FRACTION_AS_NEEDED]) && fraction < arg->fraction_out) {
|
|
arg->fraction_out = fraction;
|
|
arg->still_dropping = true;
|
|
} else if (prevent[P_DYNAMIC_DROP]) {
|
|
arg->still_dropping = true;
|
|
}
|
|
line_detail++; // to keep it the same when the loop decrements it
|
|
} else {
|
|
strategy->detail_reduced++;
|
|
}
|
|
} else {
|
|
if (pthread_mutex_lock(&db_lock) != 0) {
|
|
perror("pthread_mutex_lock");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
|
|
if (outdb != NULL) {
|
|
mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
|
|
} else if (outdir != NULL) {
|
|
dir_write_tile(outdir, z, tx, ty, compressed);
|
|
}
|
|
|
|
if (pthread_mutex_unlock(&db_lock) != 0) {
|
|
perror("pthread_mutex_unlock");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
|
|
return count;
|
|
}
|
|
} else {
|
|
return count;
|
|
}
|
|
}
|
|
|
|
fprintf(stderr, "could not make tile %d/%u/%u small enough\n", z, tx, ty);
|
|
return -1;
|
|
}
|
|
|
|
struct task {
|
|
int fileno = 0;
|
|
struct task *next = NULL;
|
|
};
|
|
|
|
void *run_thread(void *vargs) {
|
|
write_tile_args *arg = (write_tile_args *) vargs;
|
|
struct task *task;
|
|
|
|
for (task = arg->tasks; task != NULL; task = task->next) {
|
|
int j = task->fileno;
|
|
|
|
if (arg->geomfd[j] < 0) {
|
|
// only one source file for zoom level 0
|
|
continue;
|
|
}
|
|
if (arg->geom_size[j] == 0) {
|
|
continue;
|
|
}
|
|
|
|
// printf("%lld of geom_size\n", (long long) geom_size[j]);
|
|
|
|
FILE *geom = fdopen(arg->geomfd[j], "rb");
|
|
if (geom == NULL) {
|
|
perror("open geom");
|
|
exit(EXIT_OPEN);
|
|
}
|
|
|
|
std::atomic<long long> geompos(0);
|
|
long long prevgeom = 0;
|
|
|
|
while (1) {
|
|
int z;
|
|
unsigned x, y;
|
|
|
|
if (!deserialize_int_io(geom, &z, &geompos)) {
|
|
break;
|
|
}
|
|
deserialize_uint_io(geom, &x, &geompos);
|
|
deserialize_uint_io(geom, &y, &geompos);
|
|
|
|
arg->wrote_zoom = z;
|
|
|
|
// fprintf(stderr, "%d/%u/%u\n", z, x, y);
|
|
|
|
long long len = write_tile(geom, &geompos, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->fraction, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy);
|
|
|
|
if (len < 0) {
|
|
int *err = &arg->err;
|
|
*err = z - 1;
|
|
return err;
|
|
}
|
|
|
|
if (pthread_mutex_lock(&var_lock) != 0) {
|
|
perror("pthread_mutex_lock");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
|
|
if (z == arg->maxzoom) {
|
|
if (len > *arg->most) {
|
|
*arg->midx = x;
|
|
*arg->midy = y;
|
|
*arg->most = len;
|
|
} else if (len == *arg->most) {
|
|
unsigned long long a = (((unsigned long long) x) << 32) | y;
|
|
unsigned long long b = (((unsigned long long) *arg->midx) << 32) | *arg->midy;
|
|
|
|
if (a < b) {
|
|
*arg->midx = x;
|
|
*arg->midy = y;
|
|
*arg->most = len;
|
|
}
|
|
}
|
|
}
|
|
|
|
*arg->along += geompos - prevgeom;
|
|
prevgeom = geompos;
|
|
|
|
if (pthread_mutex_unlock(&var_lock) != 0) {
|
|
perror("pthread_mutex_unlock");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
}
|
|
|
|
if (arg->pass == 1) {
|
|
// Since the fclose() has closed the underlying file descriptor
|
|
arg->geomfd[j] = -1;
|
|
} else {
|
|
int newfd = dup(arg->geomfd[j]);
|
|
if (newfd < 0) {
|
|
perror("dup geometry");
|
|
exit(EXIT_OPEN);
|
|
}
|
|
if (lseek(newfd, 0, SEEK_SET) < 0) {
|
|
perror("lseek geometry");
|
|
exit(EXIT_SEEK);
|
|
}
|
|
arg->geomfd[j] = newfd;
|
|
}
|
|
|
|
if (fclose(geom) != 0) {
|
|
perror("close geom");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
}
|
|
|
|
arg->running--;
|
|
return NULL;
|
|
}
|
|
|
|
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry>> &layermaps, const char *prefilter, const char *postfilter, std::map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies) {
|
|
last_progress = 0;
|
|
|
|
// The existing layermaps are one table per input thread.
|
|
// We need to add another one per *tiling* thread so that it can be
|
|
// safely changed during tiling.
|
|
size_t layermaps_off = layermaps.size();
|
|
for (size_t i = 0; i < CPUS; i++) {
|
|
layermaps.push_back(std::map<std::string, layermap_entry>());
|
|
}
|
|
|
|
// Table to map segment and layer number back to layer name
|
|
std::vector<std::vector<std::string>> layer_unmaps;
|
|
for (size_t seg = 0; seg < layermaps.size(); seg++) {
|
|
layer_unmaps.push_back(std::vector<std::string>());
|
|
|
|
for (auto a = layermaps[seg].begin(); a != layermaps[seg].end(); ++a) {
|
|
if (a->second.id >= layer_unmaps[seg].size()) {
|
|
layer_unmaps[seg].resize(a->second.id + 1);
|
|
}
|
|
layer_unmaps[seg][a->second.id] = a->first;
|
|
}
|
|
}
|
|
|
|
int z;
|
|
for (z = 0; z <= maxzoom; z++) {
|
|
std::atomic<long long> most(0);
|
|
|
|
FILE *sub[TEMP_FILES];
|
|
int subfd[TEMP_FILES];
|
|
for (size_t j = 0; j < TEMP_FILES; j++) {
|
|
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX" XSTRINGIFY(INT_MAX)) + 1];
|
|
sprintf(geomname, "%s/geom%zu.XXXXXXXX", tmpdir, j);
|
|
subfd[j] = mkstemp_cloexec(geomname);
|
|
// printf("%s\n", geomname);
|
|
if (subfd[j] < 0) {
|
|
perror(geomname);
|
|
exit(EXIT_OPEN);
|
|
}
|
|
sub[j] = fopen_oflag(geomname, "wb", O_WRONLY | O_CLOEXEC);
|
|
if (sub[j] == NULL) {
|
|
perror(geomname);
|
|
exit(EXIT_OPEN);
|
|
}
|
|
unlink(geomname);
|
|
}
|
|
|
|
size_t useful_threads = 0;
|
|
long long todo = 0;
|
|
for (size_t j = 0; j < TEMP_FILES; j++) {
|
|
todo += geom_size[j];
|
|
if (geom_size[j] > 0) {
|
|
useful_threads++;
|
|
}
|
|
}
|
|
|
|
size_t threads = CPUS;
|
|
if (threads > TEMP_FILES / 4) {
|
|
threads = TEMP_FILES / 4;
|
|
}
|
|
// XXX is it useful to divide further if we know we are skipping
|
|
// some zoom levels? Is it faster to have fewer CPUs working on
|
|
// sharding, but more deeply, or fewer CPUs, less deeply?
|
|
if (threads > useful_threads) {
|
|
threads = useful_threads;
|
|
}
|
|
|
|
// Round down to a power of 2
|
|
for (int e = 0; e < 30; e++) {
|
|
if (threads >= (1U << e) && threads < (1U << (e + 1))) {
|
|
threads = 1U << e;
|
|
break;
|
|
}
|
|
}
|
|
if (threads >= (1U << 30)) {
|
|
threads = 1U << 30;
|
|
}
|
|
if (threads < 1) {
|
|
threads = 1;
|
|
}
|
|
|
|
// Assign temporary files to threads
|
|
|
|
std::vector<struct task> tasks;
|
|
tasks.resize(TEMP_FILES);
|
|
|
|
struct dispatch {
|
|
struct task *tasks = NULL;
|
|
long long todo = 0;
|
|
struct dispatch *next = NULL;
|
|
};
|
|
std::vector<struct dispatch> dispatches;
|
|
dispatches.resize(threads);
|
|
|
|
struct dispatch *dispatch_head = &dispatches[0];
|
|
for (size_t j = 0; j < threads; j++) {
|
|
dispatches[j].tasks = NULL;
|
|
dispatches[j].todo = 0;
|
|
if (j + 1 < threads) {
|
|
dispatches[j].next = &dispatches[j + 1];
|
|
} else {
|
|
dispatches[j].next = NULL;
|
|
}
|
|
}
|
|
|
|
for (size_t j = 0; j < TEMP_FILES; j++) {
|
|
if (geom_size[j] == 0) {
|
|
continue;
|
|
}
|
|
|
|
tasks[j].fileno = j;
|
|
tasks[j].next = dispatch_head->tasks;
|
|
dispatch_head->tasks = &tasks[j];
|
|
dispatch_head->todo += geom_size[j];
|
|
|
|
struct dispatch *here = dispatch_head;
|
|
dispatch_head = dispatch_head->next;
|
|
|
|
dispatch **d;
|
|
for (d = &dispatch_head; *d != NULL; d = &((*d)->next)) {
|
|
if (here->todo < (*d)->todo) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
here->next = *d;
|
|
*d = here;
|
|
}
|
|
|
|
int err = INT_MAX;
|
|
|
|
double zoom_gamma = gamma;
|
|
unsigned long long zoom_mingap = ((1LL << (32 - z)) / 256 * cluster_distance) * ((1LL << (32 - z)) / 256 * cluster_distance);
|
|
long long zoom_minextent = 0;
|
|
double zoom_fraction = 1;
|
|
size_t zoom_tile_size = 0;
|
|
size_t zoom_feature_count = 0;
|
|
|
|
for (size_t pass = 0;; pass++) {
|
|
pthread_t pthreads[threads];
|
|
std::vector<write_tile_args> args;
|
|
args.resize(threads);
|
|
std::atomic<int> running(threads);
|
|
std::atomic<long long> along(0);
|
|
atomic_strategy strategy;
|
|
|
|
for (size_t thread = 0; thread < threads; thread++) {
|
|
args[thread].stringpool = stringpool;
|
|
args[thread].min_detail = min_detail;
|
|
args[thread].outdb = outdb; // locked with db_lock
|
|
args[thread].outdir = outdir;
|
|
args[thread].buffer = buffer;
|
|
args[thread].fname = fname;
|
|
args[thread].geomfile = sub + thread * (TEMP_FILES / threads);
|
|
args[thread].todo = todo;
|
|
args[thread].along = &along; // locked with var_lock
|
|
args[thread].gamma = zoom_gamma;
|
|
args[thread].gamma_out = zoom_gamma;
|
|
args[thread].mingap = zoom_mingap;
|
|
args[thread].mingap_out = zoom_mingap;
|
|
args[thread].minextent = zoom_minextent;
|
|
args[thread].minextent_out = zoom_minextent;
|
|
args[thread].fraction = zoom_fraction;
|
|
args[thread].fraction_out = zoom_fraction;
|
|
args[thread].tile_size_out = 0;
|
|
args[thread].feature_count_out = 0;
|
|
args[thread].child_shards = TEMP_FILES / threads;
|
|
|
|
if (z == maxzoom && maxzoom_simplification > 0) {
|
|
args[thread].simplification = maxzoom_simplification;
|
|
} else {
|
|
args[thread].simplification = simplification;
|
|
}
|
|
|
|
args[thread].geomfd = geomfd;
|
|
args[thread].geom_size = geom_size;
|
|
args[thread].midx = midx; // locked with var_lock
|
|
args[thread].midy = midy; // locked with var_lock
|
|
args[thread].maxzoom = maxzoom;
|
|
args[thread].minzoom = minzoom;
|
|
args[thread].full_detail = full_detail;
|
|
args[thread].low_detail = low_detail;
|
|
args[thread].most = &most; // locked with var_lock
|
|
args[thread].pool_off = pool_off;
|
|
args[thread].initial_x = initial_x;
|
|
args[thread].initial_y = initial_y;
|
|
args[thread].layermaps = &layermaps;
|
|
args[thread].layer_unmaps = &layer_unmaps;
|
|
args[thread].tiling_seg = thread + layermaps_off;
|
|
args[thread].prefilter = prefilter;
|
|
args[thread].postfilter = postfilter;
|
|
args[thread].attribute_accum = attribute_accum;
|
|
args[thread].filter = filter;
|
|
|
|
args[thread].tasks = dispatches[thread].tasks;
|
|
args[thread].running = &running;
|
|
args[thread].pass = pass;
|
|
args[thread].wrote_zoom = -1;
|
|
args[thread].still_dropping = false;
|
|
args[thread].strategy = &strategy;
|
|
|
|
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
|
|
perror("pthread_create");
|
|
exit(EXIT_PTHREAD);
|
|
}
|
|
}
|
|
|
|
bool again = false;
|
|
for (size_t thread = 0; thread < threads; thread++) {
|
|
void *retval;
|
|
|
|
if (pthread_join(pthreads[thread], &retval) != 0) {
|
|
perror("pthread_join");
|
|
}
|
|
|
|
if (retval != NULL) {
|
|
err = *((int *) retval);
|
|
}
|
|
|
|
if (args[thread].gamma_out > zoom_gamma) {
|
|
zoom_gamma = args[thread].gamma_out;
|
|
again = true;
|
|
}
|
|
if (args[thread].mingap_out > zoom_mingap) {
|
|
zoom_mingap = args[thread].mingap_out;
|
|
again = true;
|
|
}
|
|
if (args[thread].minextent_out > zoom_minextent) {
|
|
zoom_minextent = args[thread].minextent_out;
|
|
again = true;
|
|
}
|
|
if (args[thread].fraction_out < zoom_fraction) {
|
|
zoom_fraction = args[thread].fraction_out;
|
|
again = true;
|
|
}
|
|
if (args[thread].tile_size_out > zoom_tile_size) {
|
|
zoom_tile_size = args[thread].tile_size_out;
|
|
}
|
|
if (args[thread].feature_count_out > zoom_feature_count) {
|
|
zoom_feature_count = args[thread].feature_count_out;
|
|
}
|
|
|
|
// Zoom counter might be lower than reality if zooms are being skipped
|
|
if (args[thread].wrote_zoom > z) {
|
|
z = args[thread].wrote_zoom;
|
|
}
|
|
|
|
if (additional[A_EXTEND_ZOOMS] && z == maxzoom && args[thread].still_dropping && maxzoom < MAX_ZOOM) {
|
|
maxzoom++;
|
|
}
|
|
}
|
|
|
|
if ((size_t) z >= strategies.size()) {
|
|
strategies.resize(z + 1);
|
|
}
|
|
|
|
struct strategy s(strategy, zoom_tile_size, zoom_feature_count);
|
|
strategies[z] = s;
|
|
|
|
if (again) {
|
|
if (outdb != NULL) {
|
|
mbtiles_erase_zoom(outdb, z);
|
|
} else if (outdir != NULL) {
|
|
dir_erase_zoom(outdir, z);
|
|
}
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (size_t j = 0; j < TEMP_FILES; j++) {
|
|
// Can be < 0 if there is only one source file, at z0
|
|
if (geomfd[j] >= 0) {
|
|
if (close(geomfd[j]) != 0) {
|
|
perror("close geom");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
}
|
|
if (fclose(sub[j]) != 0) {
|
|
perror("close subfile");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
|
|
struct stat geomst;
|
|
if (fstat(subfd[j], &geomst) != 0) {
|
|
perror("stat geom\n");
|
|
exit(EXIT_STAT);
|
|
}
|
|
|
|
geomfd[j] = subfd[j];
|
|
geom_size[j] = geomst.st_size;
|
|
}
|
|
|
|
if (err != INT_MAX) {
|
|
return err;
|
|
}
|
|
}
|
|
|
|
for (size_t j = 0; j < TEMP_FILES; j++) {
|
|
// Can be < 0 if there is only one source file, at z0
|
|
if (geomfd[j] >= 0) {
|
|
if (close(geomfd[j]) != 0) {
|
|
perror("close geom");
|
|
exit(EXIT_CLOSE);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!quiet) {
|
|
fprintf(stderr, "\n");
|
|
}
|
|
return maxzoom;
|
|
}
|