From 2abbcc70996fef7248e8dea8b12e406f951e8f9e Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Fri, 29 Mar 2024 21:55:25 -0700 Subject: [PATCH] Progress on 128-bit indices --- geometry.cpp | 8 +++---- geometry.hpp | 7 +++--- main.cpp | 10 ++++---- main.hpp | 2 +- projection.cpp | 65 +++++++++++++++++++++++++++----------------------- projection.hpp | 16 ++++++------- serial.cpp | 28 ++++++++++++++++++---- serial.hpp | 6 ++--- tile.cpp | 8 +++---- tile.hpp | 2 +- write_json.cpp | 4 ++-- write_json.hpp | 2 +- 12 files changed, 91 insertions(+), 67 deletions(-) diff --git a/geometry.cpp b/geometry.cpp index 66cb5651..0360c08b 100644 --- a/geometry.cpp +++ b/geometry.cpp @@ -578,8 +578,8 @@ drawvec reorder_lines(const drawvec &geom) { // instead of down and to the right // so that it will coalesce better - unsigned long long l1 = encode_index(coordinate_to_encodable(geom[0].x), coordinate_to_encodable(geom[0].y)); - unsigned long long l2 = encode_index(coordinate_to_encodable(geom[geom.size() - 1].x), coordinate_to_encodable(geom[geom.size() - 1].y)); + __uint128_t l1 = encode_index(coordinate_to_encodable(geom[0].x), coordinate_to_encodable(geom[0].y)); + __uint128_t l2 = encode_index(coordinate_to_encodable(geom[geom.size() - 1].x), coordinate_to_encodable(geom[geom.size() - 1].y)); if (l1 > l2) { drawvec out; @@ -1295,11 +1295,11 @@ drawvec polygon_to_anchor(const drawvec &geom) { return drawvec(); } -drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, unsigned long long label_point) { +drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, __uint128_t label_point) { drawvec out; // anchor point in world coordinates - unsigned wx, wy; + unsigned long long wx, wy; decode_index(label_point, &wx, &wy); long long wwx = decoded_to_coordinate(wx); diff --git a/geometry.hpp b/geometry.hpp index 48c70646..ae709d82 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -8,6 +8,7 @@ #include #include #include +#include "projection.hpp" #include "jsonpull/jsonpull.h" #include "attribute.hpp" @@ -24,9 +25,9 @@ // at the cost, apparently, of a 0.7% increase in running time // for packing and unpacking. struct draw { - long long x : 40; + long long x : GLOBAL_DETAIL + 2; signed char op; - long long y : 40; + long long y : GLOBAL_DETAIL + 2; signed char necessary; draw(int nop, long long nx, long long ny) @@ -88,7 +89,7 @@ void check_polygon(drawvec &geom); double get_area(const drawvec &geom, size_t i, size_t j); double get_mp_area(drawvec &geom); drawvec polygon_to_anchor(const drawvec &geom); -drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, unsigned long long label_point); +drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, __uint128_t label_point); drawvec simple_clip_poly(drawvec &geom, int z, int buffer, drawvec &shared_nodes, bool prevent_simplify_shared_nodes); drawvec simple_clip_poly(drawvec &geom, long long x1, long long y1, long long x2, long long y2, bool prevent_simplify_shared_nodes); diff --git a/main.cpp b/main.cpp index bf31232b..ec31a94f 100644 --- a/main.cpp +++ b/main.cpp @@ -283,7 +283,7 @@ static void insert(struct mergelist *m, struct mergelist **head, unsigned char * struct drop_state { double gap; - unsigned long long previndex; + __uint128_t previndex; double interval; double seq; // floating point because interval is }; @@ -822,7 +822,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split for (size_t a = 0; a < indexst.st_size / sizeof(struct index); a++) { struct index ix = indexmap[a]; - unsigned long long which = (ix.ix << prefix) >> (64 - splitbits); + unsigned long long which = (ix.ix << prefix) >> (64 - splitbits); // XX 128 long long pos = sub_geompos[which]; fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], &sub_geompos[which], "geom"); @@ -2487,7 +2487,7 @@ std::pair read_input(std::vector &sources, char *fname, i long long count; long long fullcount; double gap; - unsigned long long previndex; + __uint128_t previndex; } tile[MAX_ZOOM + 1], max[MAX_ZOOM + 1]; { @@ -2502,7 +2502,7 @@ std::pair read_input(std::vector &sources, char *fname, i long long ip; for (ip = 0; ip < indices; ip++) { - unsigned xx, yy; + unsigned long long xx, yy; decode_index(map[ip].ix, &xx, &yy); long long gxx = decoded_to_coordinate(xx); @@ -2687,7 +2687,7 @@ std::pair read_input(std::vector &sources, char *fname, i if (drop_denser > 0) { std::vector ddv; - unsigned long long previndex = 0; + __uint128_t previndex = 0; for (long long ip = 0; ip < indices; ip++) { if (map[ip].t == VT_POINT || diff --git a/main.hpp b/main.hpp index 25695e2f..f5907623 100644 --- a/main.hpp +++ b/main.hpp @@ -13,7 +13,7 @@ struct index { long long start = 0; long long end = 0; - unsigned long long ix = 0; + __int128_t ix = 0; short segment = 0; unsigned short t : 2; unsigned long long seq : (64 - 18); // pack with segment and t to stay in 32 bytes diff --git a/projection.cpp b/projection.cpp index e2262b17..71d9e8ce 100644 --- a/projection.cpp +++ b/projection.cpp @@ -9,8 +9,8 @@ #define UINT_BITS 32 -unsigned long long (*encode_index)(unsigned int wx, unsigned int wy) = NULL; -void (*decode_index)(unsigned long long index, unsigned *wx, unsigned *wy) = NULL; +__uint128_t (*encode_index)(unsigned long long wx, unsigned long long wy) = NULL; +void (*decode_index)(__uint128_t index, unsigned long long *wx, unsigned long long *wy) = NULL; struct projection projections[] = { {"EPSG:4326", lonlat2tile, tile2lonlat, "urn:ogc:def:crs:OGC:1.3:CRS84"}, @@ -107,11 +107,11 @@ void tiletoepsg3857(long long ix, long long iy, int zoom, double *ox, double *oy // https://en.wikipedia.org/wiki/Hilbert_curve -void hilbert_rot(unsigned long long n, unsigned *x, unsigned *y, unsigned long long rx, unsigned long long ry) { +void hilbert_rot(__uint128_t s, unsigned long long *x, unsigned long long *y, unsigned long long rx, unsigned long long ry) { if (ry == 0) { if (rx == 1) { - *x = n - 1 - *x; - *y = n - 1 - *y; + *x = s - 1 - *x; + *y = s - 1 - *y; } unsigned t = *x; @@ -120,11 +120,11 @@ void hilbert_rot(unsigned long long n, unsigned *x, unsigned *y, unsigned long l } } -unsigned long long hilbert_xy2d(unsigned long long n, unsigned x, unsigned y) { - unsigned long long d = 0; +__uint128_t hilbert_xy2d(__uint128_t n, unsigned long long x, unsigned long long y) { + __uint128_t d = 0; unsigned long long rx, ry; - for (unsigned long long s = n / 2; s > 0; s /= 2) { + for (__uint128_t s = n / 2; s > 0; s /= 2) { rx = (x & s) != 0; ry = (y & s) != 0; @@ -135,12 +135,12 @@ unsigned long long hilbert_xy2d(unsigned long long n, unsigned x, unsigned y) { return d; } -void hilbert_d2xy(unsigned long long n, unsigned long long d, unsigned *x, unsigned *y) { +void hilbert_d2xy(__uint128_t n, __uint128_t d, unsigned long long *x, unsigned long long *y) { unsigned long long rx, ry; - unsigned long long t = d; + __uint128_t t = d; *x = *y = 0; - for (unsigned long long s = 1; s < n; s *= 2) { + for (__uint128_t s = 1; s < n; s *= 2) { rx = 1 & (t / 2); ry = 1 & (t ^ rx); hilbert_rot(s, x, y, rx, ry); @@ -150,22 +150,22 @@ void hilbert_d2xy(unsigned long long n, unsigned long long d, unsigned *x, unsig } } -unsigned long long encode_hilbert(unsigned int wx, unsigned int wy) { - return hilbert_xy2d(1LL << UINT_BITS, wx, wy); +__uint128_t encode_hilbert(unsigned long long wx, unsigned long long wy) { + return hilbert_xy2d(1LL << GLOBAL_DETAIL, wx, wy); } -void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy) { - hilbert_d2xy(1LL << UINT_BITS, index, wx, wy); +void decode_hilbert(__uint128_t index, unsigned long long *wx, unsigned long long *wy) { + hilbert_d2xy(1LL << GLOBAL_DETAIL, index, wx, wy); } -unsigned long long encode_quadkey(unsigned int wx, unsigned int wy) { - unsigned long long out = 0; +__uint128_t encode_quadkey(unsigned long long wx, unsigned long long wy) { + __uint128_t out = 0; int i; - for (i = 0; i < UINT_BITS; i++) { - unsigned long long v = ((wx >> (UINT_BITS - (i + 1))) & 1) << 1; - v |= (wy >> (UINT_BITS - (i + 1))) & 1; - v = v << (64 - 2 * (i + 1)); + for (i = 0; i < GLOBAL_DETAIL; i++) { + unsigned long long v = ((wx >> (GLOBAL_DETAIL - (i + 1))) & 1) << 1; + v |= (wy >> (GLOBAL_DETAIL - (i + 1))) & 1; + v = v << (2 * GLOBAL_DETAIL - 2 * (i + 1)); out |= v; } @@ -176,15 +176,20 @@ unsigned long long encode_quadkey(unsigned int wx, unsigned int wy) { static std::atomic decodex[256]; static std::atomic decodey[256]; -void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy) { +void decode_quadkey(__uint128_t index, unsigned long long *wx, unsigned long long *wy) { static std::atomic initialized(0); if (!initialized) { + if (GLOBAL_DETAIL % 8 != 0) { + fprintf(stderr, "GLOBAL_DETAIL %d is not a multiple of 8\n", GLOBAL_DETAIL); + exit(EXIT_IMPOSSIBLE); + } + for (size_t ix = 0; ix < 256; ix++) { size_t xx = 0, yy = 0; - for (size_t i = 0; i < UINT_BITS; i++) { - xx |= ((ix >> (64 - 2 * (i + 1) + 1)) & 1) << (UINT_BITS - (i + 1)); - yy |= ((ix >> (64 - 2 * (i + 1) + 0)) & 1) << (UINT_BITS - (i + 1)); + for (size_t i = 0; i < GLOBAL_DETAIL; i++) { + xx |= ((ix >> (64 - 2 * (i + 1) + 1)) & 1) << (GLOBAL_DETAIL - (i + 1)); + yy |= ((ix >> (64 - 2 * (i + 1) + 0)) & 1) << (GLOBAL_DETAIL - (i + 1)); } decodex[ix] = xx; @@ -196,18 +201,18 @@ void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy) { *wx = *wy = 0; - for (size_t i = 0; i < 8; i++) { + for (size_t i = 0; i < GLOBAL_DETAIL / 8; i++) { *wx |= ((unsigned) decodex[(index >> (8 * i)) & 0xFF]) << (4 * i); *wy |= ((unsigned) decodey[(index >> (8 * i)) & 0xFF]) << (4 * i); } } -unsigned coordinate_to_encodable(long long coord) { - return (unsigned) (coord / (1LL << (GLOBAL_DETAIL - UINT_BITS))); +unsigned long long coordinate_to_encodable(long long coord) { + return coord; } -long long decoded_to_coordinate(unsigned coord) { - return ((long long) coord) * (1LL << (GLOBAL_DETAIL - UINT_BITS)); +long long decoded_to_coordinate(unsigned long long coord) { + return coord; } void set_projection_or_exit(const char *optarg) { diff --git a/projection.hpp b/projection.hpp index 789849ec..9bf3301b 100644 --- a/projection.hpp +++ b/projection.hpp @@ -19,16 +19,16 @@ struct projection { extern struct projection *projection; extern struct projection projections[]; -extern unsigned long long (*encode_index)(unsigned int wx, unsigned int wy); -extern void (*decode_index)(unsigned long long index, unsigned *wx, unsigned *wy); +extern __uint128_t (*encode_index)(unsigned long long wx, unsigned long long wy); +extern void (*decode_index)(__uint128_t index, unsigned long long *wx, unsigned long long *wy); -unsigned long long encode_quadkey(unsigned int wx, unsigned int wy); -void decode_quadkey(unsigned long long index, unsigned *wx, unsigned *wy); +__uint128_t encode_quadkey(unsigned long long wx, unsigned long long wy); +void decode_quadkey(__uint128_t index, unsigned long long *wx, unsigned long long *wy); -unsigned long long encode_hilbert(unsigned int wx, unsigned int wy); -void decode_hilbert(unsigned long long index, unsigned *wx, unsigned *wy); +__uint128_t encode_hilbert(unsigned long long wx, unsigned long long wy); +void decode_hilbert(__uint128_t index, unsigned long long *wx, unsigned long long *wy); -unsigned coordinate_to_encodable(long long coord); -long long decoded_to_coordinate(unsigned coord); +unsigned long long coordinate_to_encodable(long long coord); +long long decoded_to_coordinate(unsigned long long coord); #endif diff --git a/serial.cpp b/serial.cpp index f3d088ec..4e0f6d72 100644 --- a/serial.cpp +++ b/serial.cpp @@ -64,6 +64,11 @@ void serialize_ulong_long(FILE *out, unsigned long long zigzag, std::atomic *fpos, const char *fname) { + serialize_ulong_long(out, (unsigned long long) zigzag, fpos, fname); + serialize_ulong_long(out, (unsigned long long) (zigzag >> 64), fpos, fname); +} + void serialize_byte(FILE *out, signed char n, std::atomic *fpos, const char *fname) { fwrite_check(&n, sizeof(signed char), 1, out, fpos, fname); } @@ -92,6 +97,11 @@ void serialize_ulong_long(std::string &out, unsigned long long zigzag) { out.append(buf, s - buf); } +void serialize_u128(std::string &out, __uint128_t zigzag) { + serialize_ulong_long(out, (unsigned long long) zigzag); + serialize_ulong_long(out, (unsigned long long) (zigzag >> 64)); +} + void serialize_long_long(std::string &out, long long n) { unsigned long long zigzag = protozero::encode_zigzag64(n); @@ -142,6 +152,14 @@ void deserialize_ulong_long(const char **f, unsigned long long *zigzag) { } } +void deserialize_u128(const char **f, __uint128_t *zigzag) { + unsigned long long tmp, tmp2; + deserialize_ulong_long(f, &tmp); + deserialize_ulong_long(f, &tmp2); + + *zigzag = tmp | ((__uint128_t) tmp2) << 64; +} + void deserialize_uint(const char **f, unsigned *n) { unsigned long long v; deserialize_ulong_long(f, &v); @@ -209,10 +227,10 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) { write_geometry(sf->geometry, s, wx, wy); if (sf->index != 0) { - serialize_ulong_long(s, sf->index); + serialize_u128(s, sf->index); } if (sf->label_point != 0) { - serialize_ulong_long(s, sf->label_point); + serialize_u128(s, sf->label_point); } if (sf->extent != 0) { serialize_long_long(s, sf->extent); @@ -264,10 +282,10 @@ serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigne sf.geometry = decode_geometry(&cp, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]); if (sf.layer & (1 << FLAG_INDEX)) { - deserialize_ulong_long(&cp, &sf.index); + deserialize_u128(&cp, &sf.index); } if (sf.layer & (1 << FLAG_LABEL_POINT)) { - deserialize_ulong_long(&cp, &sf.label_point); + deserialize_u128(&cp, &sf.label_point); } if (sf.layer & (1 << FLAG_EXTENT)) { deserialize_long_long(&cp, &sf.extent); @@ -682,7 +700,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std:: *(sst->area_sum) += extent; } - unsigned long long bbox_index; + __uint128_t bbox_index; long long midx, midy; if (sf.t == VT_POINT) { diff --git a/serial.hpp b/serial.hpp index 2e8ed3d5..ae2ff4a8 100644 --- a/serial.hpp +++ b/serial.hpp @@ -67,8 +67,8 @@ struct serial_feature { int tippecanoe_maxzoom = -1; drawvec geometry = drawvec(); - unsigned long long index = 0; - unsigned long long label_point = 0; + __uint128_t index = 0; + __uint128_t label_point = 0; long long extent = 0; // These fields are not directly serialized, but are used @@ -266,7 +266,7 @@ struct node { // this is in quadkey coordinates so that the nodes for each tile // will be adjacent in memory, reducing potential thrashing during // the binary search. - unsigned long long index; + __uint128_t index; }; int nodecmp(const void *void1, const void *void2); diff --git a/tile.cpp b/tile.cpp index df7f2022..e346c68b 100644 --- a/tile.cpp +++ b/tile.cpp @@ -705,7 +705,7 @@ static void *simplification_worker(void *v) { // get rid of the --gamma option. It does something with the feature spacing to calculate // whether each feature should be kept or is in a dense enough context that it should // be dropped -int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap) { +int manage_gap(__uint128_t index, __uint128_t *previndex, double scale, double gamma, double *gap) { if (gamma > 0) { if (*gap > 0) { if (index == *previndex) { @@ -1525,8 +1525,8 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch long long count = 0; double accum_area = 0; - unsigned long long previndex = 0, density_previndex = 0, merge_previndex = 0; - unsigned long long extent_previndex = 0; + __uint128_t previndex = 0, density_previndex = 0, merge_previndex = 0; + __uint128_t extent_previndex = 0; double scale = (double) (1LL << (64 - 2 * (z + 8))); double gap = 0, density_gap = 0; double spacing = 0; @@ -1870,7 +1870,7 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch } } - unsigned long long sfindex = sf.index; + __uint128_t sfindex = sf.index; if (sf.geometry.size() > 0) { if (lead_features_count > max_tile_size) { diff --git a/tile.hpp b/tile.hpp index 868f55a4..0fedd012 100644 --- a/tile.hpp +++ b/tile.hpp @@ -57,6 +57,6 @@ struct strategy { int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic *midx, std::atomic *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *pool_off, long long *initial_x, long long *initial_y, double simplification, double maxzoom_simplification, std::vector > &layermap, const char *prefilter, const char *postfilter, std::unordered_map const *attribute_accum, struct json_object *filter, std::vector &strategies, int iz, struct node *shared_nodes_map, size_t nodepos, int basezoom, double droprate, std::vector const &unidecode_data); -int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); +int manage_gap(__uint128_t index, __uint128_t *previndex, double scale, double gamma, double *gap); #endif diff --git a/write_json.cpp b/write_json.cpp index bdf1e852..909e1b10 100644 --- a/write_json.cpp +++ b/write_json.cpp @@ -268,7 +268,7 @@ void write_coords(json_writer &state, lonlat const &ll, double scale) { } } -void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool write_dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state, double scale) { +void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool write_dropped, __uint128_t index, long long sequence, long long extent, bool complain, json_writer &state, double scale) { for (size_t f = 0; f < layer.features.size(); f++) { mvt_feature const &feat = layer.features[f]; @@ -305,7 +305,7 @@ void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y if (index != 0) { state.json_write_string("index"); - state.json_write_unsigned(index); + state.json_write_unsigned(index); // XXX 128 } if (sequence != 0) { diff --git a/write_json.hpp b/write_json.hpp index 523b027b..7f83a25d 100644 --- a/write_json.hpp +++ b/write_json.hpp @@ -61,7 +61,7 @@ struct json_writer { void adds(std::string const &s); }; -void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, unsigned long long index, long long sequence, long long extent, bool complain, json_writer &state, double scale); +void layer_to_geojson(mvt_layer const &layer, unsigned z, unsigned x, unsigned y, bool comma, bool name, bool zoom, bool dropped, __uint128_t index, long long sequence, long long extent, bool complain, json_writer &state, double scale); void fprintq(FILE *f, const char *s); #endif