diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Progress on 128-bit indices

Browse Source
This commit is contained in:
Erica Fischer
2024-03-29 21:55:25 -07:00
parent 054becc58f
commit 2abbcc7099
12 changed files with 91 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
geometry.cpp
+4 -4
View File
@@ -578,8 +578,8 @@ drawvec reorder_lines(const drawvec &geom) {
// instead of down and to the right
// so that it will coalesce better
unsigned long long l1 = encode_index(coordinate_to_encodable(geom[0].x), coordinate_to_encodable(geom[0].y));
unsigned long long l2 = encode_index(coordinate_to_encodable(geom[geom.size() - 1].x), coordinate_to_encodable(geom[geom.size() - 1].y));
__uint128_t l1 = encode_index(coordinate_to_encodable(geom[0].x), coordinate_to_encodable(geom[0].y));
__uint128_t l2 = encode_index(coordinate_to_encodable(geom[geom.size() - 1].x), coordinate_to_encodable(geom[geom.size() - 1].y));
if (l1 > l2) {
drawvec out;
@@ -1295,11 +1295,11 @@ drawvec polygon_to_anchor(const drawvec &geom) {
return drawvec();
}
drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, unsigned long long label_point) {
drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, __uint128_t label_point) {
drawvec out;
// anchor point in world coordinates
unsigned wx, wy;
unsigned long long wx, wy;
decode_index(label_point, &wx, &wy);
long long wwx = decoded_to_coordinate(wx);
geometry.hpp
+4 -3
View File
@@ -8,6 +8,7 @@
#include <sqlite3.h>
#include <stdio.h>
#include <mvt.hpp>
#include "projection.hpp"
#include "jsonpull/jsonpull.h"
#include "attribute.hpp"
@@ -24,9 +25,9 @@
// at the cost, apparently, of a 0.7% increase in running time
// for packing and unpacking.
struct draw {
long long x : 40;
long long x : GLOBAL_DETAIL + 2;
signed char op;
long long y : 40;
long long y : GLOBAL_DETAIL + 2;
signed char necessary;
draw(int nop, long long nx, long long ny)
@@ -88,7 +89,7 @@ void check_polygon(drawvec &geom);
double get_area(const drawvec &geom, size_t i, size_t j);
double get_mp_area(drawvec &geom);
drawvec polygon_to_anchor(const drawvec &geom);
drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, unsigned long long label_point);
drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, __uint128_t label_point);
drawvec simple_clip_poly(drawvec &geom, int z, int buffer, drawvec &shared_nodes, bool prevent_simplify_shared_nodes);
drawvec simple_clip_poly(drawvec &geom, long long x1, long long y1, long long x2, long long y2, bool prevent_simplify_shared_nodes);
main.cpp
+5 -5
View File
@@ -283,7 +283,7 @@ static void insert(struct mergelist *m, struct mergelist **head, unsigned char *
struct drop_state {
double gap;
unsigned long long previndex;
__uint128_t previndex;
double interval;
double seq; // floating point because interval is
};
@@ -822,7 +822,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) {
struct index ix = indexmap[a];
unsigned long long which = (ix.ix << prefix) >> (64 - splitbits);
unsigned long long which = (ix.ix << prefix) >> (64 - splitbits); // XX 128
long long pos = sub_geompos[which];
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], &sub_geompos[which], "geom");
@@ -2487,7 +2487,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
long long count;
long long fullcount;
double gap;
unsigned long long previndex;
__uint128_t previndex;
} tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1];
{
@@ -2502,7 +2502,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
long long ip;
for (ip = 0; ip < indices; ip++) {
unsigned xx, yy;
unsigned long long xx, yy;
decode_index(map[ip].ix, &xx, &yy);
long long gxx = decoded_to_coordinate(xx);
@@ -2687,7 +2687,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
if (drop_denser > 0) {
std::vector<drop_densest> ddv;
unsigned long long previndex = 0;
__uint128_t previndex = 0;
for (long long ip = 0; ip < indices; ip++) {
if (map[ip].t == VT_POINT ||
main.hpp
+1 -1
View File
@@ -13,7 +13,7 @@
struct index {
long long start = 0;
long long end = 0;
unsigned long long ix = 0;
__int128_t ix = 0;
short segment = 0;
unsigned short t : 2;
unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes
projection.cpp
+35 -30
View File
@@ -9,8 +9,8 @@
#define UINT_BITS 32
unsigned long long (*encode_index)(unsigned int wx, unsigned int wy) = NULL;
void (*decode_index)(unsigned long long index, unsigned *wx, unsigned *wy) = NULL;
__uint128_t (*encode_index)(unsigned long long wx, unsigned long long wy) = NULL;
void (*decode_index)(__uint128_t index, unsigned long long *wx, unsigned long long *wy) = NULL;
struct projection projections[] = {
{"EPSG:4326", lonlat2tile, tile2lonlat, "urn:ogc:def:crs:OGC:1.3:CRS84"},
@@ -107,11 +107,11 @@ void tiletoepsg3857(long long ix, long long iy, int zoom, double *ox, double *oy
// https://en.wikipedia.org/wiki/Hilbert_curve
void hilbert_rot(unsigned long long n, unsigned *x, unsigned *y, unsigned long long rx, unsigned long long ry) {
void hilbert_rot(__uint128_t s, unsigned long long *x, unsigned long long *y, unsigned long long rx, unsigned long long ry) {
if (ry == 0) {
if (rx == 1) {
*x = n - 1 - *x;
*y = n - 1 - *y;
*x = s - 1 - *x;
*y = s - 1 - *y;
}
unsigned t = *x;
@@ -120,11 +120,11 @@ void hilbert_rot(unsigned long long n, unsigned *x, unsigned *y, unsigned long l
}
}
unsigned long long hilbert_xy2d(unsigned long long n, unsigned x, unsigned y) {
unsigned long long d = 0;
__uint128_t hilbert_xy2d(__uint128_t n, unsigned long long x, unsigned long long y) {
__uint128_t d = 0;
unsigned long long rx, ry;
for (unsigned long long s = n / 2; s > 0; s /= 2) {
for (__uint128_t s = n / 2; s > 0; s /= 2) {
rx = (x & s) != 0;
ry = (y & s) != 0;
@@ -135,12 +135,12 @@ unsigned long long hilbert_xy2d(unsigned long long n, unsigned x, unsigned y) {
return d;
}
void hilbert_d2xy(unsigned long long n, unsigned long long d, unsigned *x, unsigned *y) {
void hilbert_d2xy(__uint128_t n, __uint128_t d, unsigned long long *x, unsigned long long *y) {
unsigned long long rx, ry;
unsigned long long t = d;
__uint128_t t = d;
*x = *y = 0;
for (unsigned long long s = 1; s < n; s *= 2) {
for (__uint128_t s = 1; s < n; s *= 2) {
rx = 1 & (t / 2);
ry = 1 & (t ^ rx);
hilbert_rot(s, x, y, rx, ry);
@@ -150,22 +150,22 @@ void hilbert_d2xy(unsigned long long n, unsigned long long d, unsigned *x, unsig
}
}
unsigned long long encode_hilbert(unsigned int wx, unsigned int wy) {
return hilbert_xy2d(1LL << UINT_BITS, wx, wy);
__uint128_t encode_hilbert(unsigned long long wx, unsigned long long wy) {
return hilbert_xy2d(1LL << GLOBAL_DETAIL, wx, wy);
}
void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy) {
hilbert_d2xy(1LL << UINT_BITS, index, wx, wy);
void decode_hilbert(__uint128_t index, unsigned long long *wx, unsigned long long *wy) {
hilbert_d2xy(1LL << GLOBAL_DETAIL, index, wx, wy);
}
unsigned long long encode_quadkey(unsigned int wx, unsigned int wy) {
unsigned long long out = 0;
__uint128_t encode_quadkey(unsigned long long wx, unsigned long long wy) {
__uint128_t out = 0;
int i;
for (i = 0; i < UINT_BITS; i++) {
unsigned long long v = ((wx >> (UINT_BITS - (i + 1))) & 1) << 1;
v |= (wy >> (UINT_BITS - (i + 1))) & 1;
v = v << (64 - 2 * (i + 1));
for (i = 0; i < GLOBAL_DETAIL; i++) {
unsigned long long v = ((wx >> (GLOBAL_DETAIL - (i + 1))) & 1) << 1;
v |= (wy >> (GLOBAL_DETAIL - (i + 1))) & 1;
v = v << (2 * GLOBAL_DETAIL - 2 * (i + 1));
out |= v;
}
@@ -176,15 +176,20 @@ unsigned long long encode_quadkey(unsigned int wx, unsigned int wy) {
static std::atomic<unsigned char> decodex[256];
static std::atomic<unsigned char> decodey[256];
void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy) {
void decode_quadkey(__uint128_t index, unsigned long long *wx, unsigned long long *wy) {
static std::atomic<int> initialized(0);
if (!initialized) {
if (GLOBAL_DETAIL % 8 != 0) {
fprintf(stderr, "GLOBAL_DETAIL %d is not a multiple of 8\n", GLOBAL_DETAIL);
exit(EXIT_IMPOSSIBLE);
}
for (size_t ix = 0; ix < 256; ix++) {
size_t xx = 0, yy = 0;
for (size_t i = 0; i < UINT_BITS; i++) {
xx |= ((ix >> (64 - 2 * (i + 1) + 1)) & 1) << (UINT_BITS - (i + 1));
yy |= ((ix >> (64 - 2 * (i + 1) + 0)) & 1) << (UINT_BITS - (i + 1));
for (size_t i = 0; i < GLOBAL_DETAIL; i++) {
xx |= ((ix >> (64 - 2 * (i + 1) + 1)) & 1) << (GLOBAL_DETAIL - (i + 1));
yy |= ((ix >> (64 - 2 * (i + 1) + 0)) & 1) << (GLOBAL_DETAIL - (i + 1));
}
decodex[ix] = xx;
@@ -196,18 +201,18 @@ void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy) {
*wx = *wy = 0;
for (size_t i = 0; i < 8; i++) {
for (size_t i = 0; i < GLOBAL_DETAIL / 8; i++) {
*wx |= ((unsigned) decodex[(index >> (8 * i)) & 0xFF]) << (4 * i);
*wy |= ((unsigned) decodey[(index >> (8 * i)) & 0xFF]) << (4 * i);
}
}
unsigned coordinate_to_encodable(long long coord) {
return (unsigned) (coord / (1LL << (GLOBAL_DETAIL - UINT_BITS)));
unsigned long long coordinate_to_encodable(long long coord) {
return coord;
}
long long decoded_to_coordinate(unsigned coord) {
return ((long long) coord) * (1LL << (GLOBAL_DETAIL - UINT_BITS));
long long decoded_to_coordinate(unsigned long long coord) {
return coord;
}
void set_projection_or_exit(const char *optarg) {
projection.hpp
+8 -8
View File
@@ -19,16 +19,16 @@ struct projection {
extern struct projection *projection;
extern struct projection projections[];
extern unsigned long long (*encode_index)(unsigned int wx, unsigned int wy);
extern void (*decode_index)(unsigned long long index, unsigned *wx, unsigned *wy);
extern __uint128_t (*encode_index)(unsigned long long wx, unsigned long long wy);
extern void (*decode_index)(__uint128_t index, unsigned long long *wx, unsigned long long *wy);
unsigned long long encode_quadkey(unsigned int wx, unsigned int wy);
void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy);
__uint128_t encode_quadkey(unsigned long long wx, unsigned long long wy);
void decode_quadkey(__uint128_t index, unsigned long long *wx, unsigned long long *wy);
unsigned long long encode_hilbert(unsigned int wx, unsigned int wy);
void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy);
__uint128_t encode_hilbert(unsigned long long wx, unsigned long long wy);
void decode_hilbert(__uint128_t index, unsigned long long *wx, unsigned long long *wy);
unsigned coordinate_to_encodable(long long coord);
long long decoded_to_coordinate(unsigned coord);
unsigned long long coordinate_to_encodable(long long coord);
long long decoded_to_coordinate(unsigned long long coord);
#endif
serial.cpp
+23 -5
View File
@@ -64,6 +64,11 @@ void serialize_ulong_long(FILE *out, unsigned long long zigzag, std::atomic<long
fwrite_check(buf, sizeof(char), s - buf, out, fpos, fname);
}
void serialize_u128(FILE *out, __uint128_t zigzag, std::atomic<long long> *fpos, const char *fname) {
serialize_ulong_long(out, (unsigned long long) zigzag, fpos, fname);
serialize_ulong_long(out, (unsigned long long) (zigzag >> 64), fpos, fname);
}
void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname) {
fwrite_check(&n, sizeof(signed char), 1, out, fpos, fname);
}
@@ -92,6 +97,11 @@ void serialize_ulong_long(std::string &out, unsigned long long zigzag) {
out.append(buf, s - buf);
}
void serialize_u128(std::string &out, __uint128_t zigzag) {
serialize_ulong_long(out, (unsigned long long) zigzag);
serialize_ulong_long(out, (unsigned long long) (zigzag >> 64));
}
void serialize_long_long(std::string &out, long long n) {
unsigned long long zigzag = protozero::encode_zigzag64(n);
@@ -142,6 +152,14 @@ void deserialize_ulong_long(const char **f, unsigned long long *zigzag) {
}
}
void deserialize_u128(const char **f, __uint128_t *zigzag) {
unsigned long long tmp, tmp2;
deserialize_ulong_long(f, &tmp);
deserialize_ulong_long(f, &tmp2);
*zigzag = tmp | ((__uint128_t) tmp2) << 64;
}
void deserialize_uint(const char **f, unsigned *n) {
unsigned long long v;
deserialize_ulong_long(f, &v);
@@ -209,10 +227,10 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
write_geometry(sf->geometry, s, wx, wy);
if (sf->index != 0) {
serialize_ulong_long(s, sf->index);
serialize_u128(s, sf->index);
}
if (sf->label_point != 0) {
serialize_ulong_long(s, sf->label_point);
serialize_u128(s, sf->label_point);
}
if (sf->extent != 0) {
serialize_long_long(s, sf->extent);
@@ -264,10 +282,10 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne
sf.geometry = decode_geometry(&cp, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
if (sf.layer & (1 << FLAG_INDEX)) {
deserialize_ulong_long(&cp, &sf.index);
deserialize_u128(&cp, &sf.index);
}
if (sf.layer & (1 << FLAG_LABEL_POINT)) {
deserialize_ulong_long(&cp, &sf.label_point);
deserialize_u128(&cp, &sf.label_point);
}
if (sf.layer & (1 << FLAG_EXTENT)) {
deserialize_long_long(&cp, &sf.extent);
@@ -682,7 +700,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::
*(sst->area_sum) += extent;
}
unsigned long long bbox_index;
__uint128_t bbox_index;
long long midx, midy;
if (sf.t == VT_POINT) {
serial.hpp
+3 -3
View File
@@ -67,8 +67,8 @@ struct serial_feature {
int tippecanoe_maxzoom = -1;
drawvec geometry = drawvec();
unsigned long long index = 0;
unsigned long long label_point = 0;
__uint128_t index = 0;
__uint128_t label_point = 0;
long long extent = 0;
// These fields are not directly serialized, but are used
@@ -266,7 +266,7 @@ struct node {
// this is in quadkey coordinates so that the nodes for each tile
// will be adjacent in memory, reducing potential thrashing during
// the binary search.
unsigned long long index;
__uint128_t index;
};
int nodecmp(const void *void1, const void *void2);
tile.cpp
+4 -4
View File
@@ -705,7 +705,7 @@ static void *simplification_worker(void *v) {
// get rid of the --gamma option. It does something with the feature spacing to calculate
// whether each feature should be kept or is in a dense enough context that it should
// be dropped
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap) {
int manage_gap(__uint128_t index, __uint128_t *previndex, double scale, double gamma, double *gap) {
if (gamma > 0) {
if (*gap > 0) {
if (index == *previndex) {
@@ -1525,8 +1525,8 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
long long count = 0;
double accum_area = 0;
unsigned long long previndex = 0, density_previndex = 0, merge_previndex = 0;
unsigned long long extent_previndex = 0;
__uint128_t previndex = 0, density_previndex = 0, merge_previndex = 0;
__uint128_t extent_previndex = 0;
double scale = (double) (1LL << (64 - 2 * (z + 8)));
double gap = 0, density_gap = 0;
double spacing = 0;
@@ -1870,7 +1870,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
}
}
unsigned long long sfindex = sf.index;
__uint128_t sfindex = sf.index;
if (sf.geometry.size() > 0) {
if (lead_features_count > max_tile_size) {
tile.hpp
+1 -1
View File
@@ -57,6 +57,6 @@ struct strategy {
int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<unsigned> *midx, std::atomic<unsigned> *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, long long *initial_x, long long *initial_y, double simplification, double maxzoom_simplification, std::vector<std::map<std::string, layermap_entry> > &layermap, const char *prefilter, const char *postfilter, std::unordered_map<std::string, attribute_op> const *attribute_accum, struct json_object *filter, std::vector<strategy> &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector<std::string> const &unidecode_data);
int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap);
int manage_gap(__uint128_t index, __uint128_t *previndex, double scale, double gamma, double *gap);
#endif
write_json.cpp
+2 -2
View File
@@ -268,7 +268,7 @@ void write_coords(json_writer &state, lonlat const &ll, double scale) {
}
}
void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool write_dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state, double scale) {
void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool write_dropped, __uint128_t index, long long sequence, long long extent, bool complain, json_writer &state, double scale) {
for (size_t f = 0; f < layer.features.size(); f++) {
mvt_feature const &feat = layer.features[f];
@@ -305,7 +305,7 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y
if (index != 0) {
state.json_write_string("index");
state.json_write_unsigned(index);
state.json_write_unsigned(index); // XXX 128
}
if (sequence != 0) {
write_json.hpp
+1 -1
View File
@@ -61,7 +61,7 @@ struct json_writer {
void adds(std::string const &s);
};
void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state, double scale);
void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, __uint128_t index, long long sequence, long long extent, bool complain, json_writer &state, double scale);
void fprintq(FILE *f, const char *s);
#endif