diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improve tiling speed (#195)

Browse Source 
* Add a way to run tippecanoe single-threaded for profiling

* Do less work when the tilestats sample values list is already full

* Save a copy when retrieving the attribute key

* Fewer atomic operations

* Move string hashing from mbtiles to text

* Only do approximate attribute deduplication when writing tiles

* Feature dropping tests are sensitive to exact tile size

* All tile creators now create a string pool for the tile

* Features clipped away to nothing should not participate in that tile

* Revert "Only do approximate attribute deduplication when writing tiles"

This reverts commit c42b34b498.

* Also revert the related test changes

* Revert "Revert "Only do approximate attribute deduplication when writing tiles""

This reverts commit 18509876c3.

* Be more specific about the string hash function

* Use fnv1a instead of std::hash for everything

* Reduce the chance of hash collisions

* Stick a hash search on the front of the tree search in addpool

* Eliminate repeated hashing of the same string

* Switch instead of ifs in json parsing

* A few more cases to populate the hash in addpool

* Store the hash in the tree instead of recalculating

* Add explanatory comment for mysterious argument

* Fewer copies in attribute stringification

* Clean up ancient weirdness in JSON attribute stringification

* More serial_val cleanup

* Pass a serial_feature to rewrite instead of many broken-down arguments

* Get rid of the multiple geometries within `partial`

* Revert "Pass a serial_feature to rewrite instead of many broken-down arguments"

This reverts commit 6f4ab9b725.

* Goodbye, struct coalesce

* Revert "Features clipped away to nothing should not participate in that tile"

This reverts commit 124462fbdc.

* Migrating fields from partial to serial_feature

* Name reconciliation between serial_feature and partial

* Replace struct partial with an augmented serial_feature

* Fix some overzealous search-and-replace renaming

* Don't say struct so often

* Remove more of the former partial construction

* Commenting and cleaning up

* Trying again to avoid all these arguments to rewrite

* I swear I did this same thing before and it didn't work.

* More rewrite cleanup

* Exile --detect-shared-borders to its own file

* Add missing headers

* More commenting and cleanup

* More comments

* Sprinkle consts around

* Emplacing and std::moving

* More cleanup

* That shouldn't have worked after a std::move

* Don't need to allocate memory to compare keys

* Reduce use of the global string pool in tiling

* Another avoidable mvt_value construction

* Further reduction to explicit string pool passing

* These reverses are no longer optimizations

* These layernames can all be references

* Don't drag an unused layername string around with every feature

* Heed a compiler warning about potential buffer overflow

* Fix my confusion about which feature's string pool is relevant

* Avoid some unnecessary allocations in attribute accumulation

* Maybe faster serialization?

* Eliminate a comparison

* Do the same here

* Save a couple of allocations when parsing numbers in JSON

* Immediately assign features to layers instead of subdividing later

* Maintain tilestats for tippecanoe:retain_points_multiplier_sequence

* Crunch out more duplicate attribute values when writing out the tile

* Do tilestats for tippecanoe:retain_points_multiplier_first too

* Shell filters need to be real threads, even if nothing else does

* Simplify tippecanoe_minzoom/maxzoom representation

* Update version and changelog
This commit is contained in:
Erica Fischer
2024-02-07 17:50:35 -08:00
committed by GitHub
parent c5c4698161
commit e2a7a409c7
46 changed files with 22817 additions and 22778 deletions
Download Patch File Download Diff File Expand all files Collapse all files
serial.cpp
+39 -49
View File
@@ -52,25 +52,16 @@ void serialize_long_long(FILE *out, long long n, std::atomic<long long> *fpos, c
}
void serialize_ulong_long(FILE *out, unsigned long long zigzag, std::atomic<long long> *fpos, const char *fname) {
while (1) {
unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) {
b |= 0x80;
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_WRITE);
}
*fpos += 1;
zigzag >>= 7;
} else {
if (putc(b, out) == EOF) {
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
exit(EXIT_WRITE);
}
*fpos += 1;
break;
}
char buf[10]; // ceil(64 / 7)
char *s = buf;
while (zigzag >= 0x80) {
*s++ = (zigzag & 0x7F) | 0x80;
zigzag >>= 7;
}
*s++ = zigzag;
fwrite_check(buf, sizeof(char), s - buf, out, fpos, fname);
}
void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname) {
@@ -89,17 +80,16 @@ size_t fwrite_check(const void *ptr, size_t size, size_t nitems, std::string &st
}
void serialize_ulong_long(std::string &out, unsigned long long zigzag) {
while (1) {
unsigned char b = zigzag & 0x7F;
if ((zigzag >> 7) != 0) {
b |= 0x80;
out += b;
zigzag >>= 7;
} else {
out += b;
break;
}
char buf[10]; // ceil(64 / 7)
char *s = buf;
while (zigzag >= 0x80) {
*s++ = (zigzag & 0x7F) | 0x80;
zigzag >>= 7;
}
*s++ = zigzag;
out.append(buf, s - buf);
}
void serialize_long_long(std::string &out, long long n) {
@@ -122,43 +112,43 @@ void serialize_uint(std::string &out, unsigned n) {
// read from memory
void deserialize_int(char **f, int *n) {
void deserialize_int(const char **f, int *n) {
long long ll;
deserialize_long_long(f, &ll);
*n = ll;
}
void deserialize_long_long(char **f, long long *n) {
void deserialize_long_long(const char **f, long long *n) {
unsigned long long zigzag = 0;
deserialize_ulong_long(f, &zigzag);
*n = protozero::decode_zigzag64(zigzag);
}
void deserialize_ulong_long(char **f, unsigned long long *zigzag) {
void deserialize_ulong_long(const char **f, unsigned long long *zigzag) {
*zigzag = 0;
int shift = 0;
while (1) {
if ((**f & 0x80) == 0) {
*zigzag |= ((unsigned long long) **f) << shift;
*zigzag |= ((const unsigned long long) **f) << shift;
*f += 1;
shift += 7;
break;
} else {
*zigzag |= ((unsigned long long) (**f & 0x7F)) << shift;
*zigzag |= ((const unsigned long long) (**f & 0x7F)) << shift;
*f += 1;
shift += 7;
}
}
}
void deserialize_uint(char **f, unsigned *n) {
void deserialize_uint(const char **f, unsigned *n) {
unsigned long long v;
deserialize_ulong_long(f, &v);
*n = v;
}
void deserialize_byte(char **f, signed char *n) {
void deserialize_byte(const char **f, signed char *n) {
memcpy(n, *f, sizeof(signed char));
*f += sizeof(signed char);
}
@@ -199,15 +189,15 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
layer |= (sf->index != 0) << FLAG_INDEX;
layer |= (sf->extent != 0) << FLAG_EXTENT;
layer |= sf->has_id << FLAG_ID;
layer |= sf->has_tippecanoe_minzoom << FLAG_MINZOOM;
layer |= sf->has_tippecanoe_maxzoom << FLAG_MAXZOOM;
layer |= (sf->tippecanoe_minzoom != -1) << FLAG_MINZOOM;
layer |= (sf->tippecanoe_maxzoom != -1) << FLAG_MAXZOOM;
serialize_long_long(s, layer);
serialize_long_long(s, sf->seq);
if (sf->has_tippecanoe_minzoom) {
if (sf->tippecanoe_minzoom != -1) {
serialize_int(s, sf->tippecanoe_minzoom);
}
if (sf->has_tippecanoe_maxzoom) {
if (sf->tippecanoe_maxzoom != -1) {
serialize_int(s, sf->tippecanoe_maxzoom);
}
if (sf->has_id) {
@@ -240,9 +230,9 @@ std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
return s;
}
serial_feature deserialize_feature(std::string &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
serial_feature deserialize_feature(std::string const &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
serial_feature sf;
char *cp = (char *) geoms.c_str();
const char *cp = geoms.c_str();
deserialize_byte(&cp, &sf.t);
deserialize_long_long(&cp, &sf.layer);
@@ -419,7 +409,7 @@ static void add_scaled_node(struct reader *r, serialization_state *sst, draw g)
}
// called from frontends
int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::string const &layername) {
struct reader *r = &(*sst->readers)[sst->segment];
sf.bbox[0] = LLONG_MAX;
@@ -725,11 +715,11 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
sf.index = 0;
}
if (sst->layermap->count(sf.layername) == 0) {
sst->layermap->insert(std::pair<std::string, layermap_entry>(sf.layername, layermap_entry(sst->layermap->size())));
if (sst->layermap->count(layername) == 0) {
sst->layermap->emplace(layername, layermap_entry(sst->layermap->size()));
}
auto ai = sst->layermap->find(sf.layername);
auto ai = sst->layermap->find(layername);
if (ai != sst->layermap->end()) {
sf.layer = ai->second.id;
@@ -743,7 +733,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
}
}
} else {
fprintf(stderr, "Internal error: can't find layer name %s\n", sf.layername.c_str());
fprintf(stderr, "Internal error: can't find layer name %s\n", layername.c_str());
exit(EXIT_IMPOSSIBLE);
}
@@ -827,14 +817,14 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
if (!sst->filters) {
for (size_t i = 0; i < sf.full_keys.size(); i++) {
auto ts = sst->layermap->find(sf.layername);
auto ts = sst->layermap->find(layername);
add_to_tilestats(ts->second.tilestats, sf.full_keys[i], sf.full_values[i]);
}
}
for (size_t i = 0; i < sf.full_keys.size(); i++) {
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string, r->key_dedup));
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type, r->value_dedup));
}
long long geomstart = r->geompos;