diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix-polygons-with-earcut

Browse Source
This commit is contained in:
Erica Fischer
2023-12-06 15:55:04 -08:00
parent 93c7398998 d7bdbe363b
commit 074ff0519b
23 changed files with 117 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+13
View File
@@ -1,3 +1,16 @@
# 2.37.1
* Reduce maximum memory used for vertex sorting
# 2.37.0
* Speed up tile-join overzooming and make it use less memory, by not including empty child tiles in the enumeration
# 2.36.0
* Make tile-join distrust the source tilesets' metadata maxzoom and minzoom
* Add a special case in --detect-longitude-wraparound not to wrap around jumps of exactly 360°
# 2.35.0
* Fix a bug in --detect-longitude-wraparound when there are multiple rings
clip.cpp
+27 -3
View File
@@ -753,7 +753,8 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
}
std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
mvt_tile tile;
try {
@@ -767,11 +768,12 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
exit(EXIT_PROTOBUF);
}
return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress);
return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
}
std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress) {
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
mvt_tile outtile;
for (auto const &layer : tile.layers) {
@@ -890,6 +892,28 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
}
}
if (next_overzoomed_tiles != NULL) {
// will any child tiles have features in them?
// find out recursively from the tile we just made.
//
// (yes, we should keep them instead of remaking them
// later, but that first requires figuring out where to
// keep them.)
if (outtile.layers.size() > 0) {
for (size_t x = 0; x < 2; x++) {
for (size_t y = 0; y < 2; y++) {
std::string child = overzoom(outtile, nz, nx, ny,
nz + 1, nx * 2 + x, ny * 2 + y,
detail, buffer, keep, false, NULL);
if (child.size() > 0) {
next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
}
}
}
}
}
if (outtile.layers.size() > 0) {
std::string pbf = outtile.encode();
geometry.hpp
+4 -2
View File
@@ -99,10 +99,12 @@ int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, lon
double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y, double *px, double *py);
std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
int detail, int buffer, std::set<std::string> const &keep, bool do_compress);
int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
drawvec fix_by_triangulation(drawvec const &dv, int z, int detail);
main.cpp
+2 -2
View File
@@ -2041,7 +2041,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
vertex_readers.push_back(readers[i].vertexfile);
rewind(readers[i].vertexfile);
}
fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 10);
fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 20);
for (size_t i = 0; i < CPUS; i++) {
if (fclose(readers[i].vertexfile) != 0) {
@@ -2107,7 +2107,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
rewind(readers[i].nodefile);
}
fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 10);
fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 20);
for (size_t i = 0; i < CPUS; i++) {
if (fclose(readers[i].nodefile) != 0) {
overzoom.cpp
+1 -1
View File
@@ -91,7 +91,7 @@ int main(int argc, char **argv) {
exit(EXIT_FAILURE);
}
std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true);
std::string out = overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, true, NULL);
fwrite(out.c_str(), sizeof(char), out.size(), f);
fclose(f);
serial.cpp
+5 -2
View File
@@ -328,10 +328,13 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox
if (geom[i].op == VT_LINETO) {
x += offset;
if (has_prev) {
if (x - prev > (1LL << 31)) {
// jumps at least 180° but not exactly 360°,
// which in some data sets is an intentional
// line across the world
if (x - prev > (1LL << 31) && x - prev != (1LL << 32)) {
offset -= 1LL << 32;
x -= 1LL << 32;
} else if (prev - x > (1LL << 31)) {
} else if (prev - x > (1LL << 31) && prev - x != (1LL << 32)) {
offset += 1LL << 32;
x += 1LL << 32;
}
sort.cpp
+5 -1
View File
@@ -4,12 +4,16 @@
#include <vector>
#include <string>
#define MAX_MEMORY (10 * 1024 * 1024)
#define MAX_MEMORY (1024 * 1024 * 1024) // 1 GB
void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) {
std::string pivot;
FILE *fp1, *fp2;
if (mem > MAX_MEMORY) {
mem = MAX_MEMORY;
}
{
// read some elements into memory to choose a pivot from
//
tests/join-population/joined-i.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/joined-no-tile-stats.mbtiles.json
+1 -1
View File
@@ -7,7 +7,7 @@
"generator_options": "./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles '-YALAND10:Land area' '-L{\"file\": \"tests/join-population/tabblock_06001420.json\", \"description\": \"population\"}'; ./tile-join --quiet --force --no-tile-stats -o tests/join-population/joined-no-tile-stats.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles",
"json": "{\"vector_layers\":[{\"id\":\"tabblock_06001420\",\"description\":\"population\",\"minzoom\":0,\"maxzoom\":12,\"fields\":{\"ALAND10\":\"Land area\",\"AWATER10\":\"Number\",\"BLOCKCE10\":\"String\",\"COUNTYFP10\":\"String\",\"FUNCSTAT10\":\"String\",\"INTPTLAT10\":\"String\",\"INTPTLON10\":\"String\",\"MTFCC10\":\"String\",\"NAME10\":\"String\",\"STATEFP10\":\"String\",\"TRACTCE10\":\"String\",\"UACE10\":\"String\",\"UATYP10\":\"String\",\"UR10\":\"String\",\"population\":\"Mixed\"}}]}",
"maxzoom": "12",
"minzoom": "0",
"minzoom": "3",
"name": "tests/join-population/tabblock_06001420.mbtiles",
"strategies": "[{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":197},{\"tiny_polygons\":195},{\"tiny_polygons\":189},{\"tiny_polygons\":97},{\"tiny_polygons\":34},{\"tiny_polygons\":24}]",
"type": "overlay",
tests/join-population/joined-null.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json
+1 -1
View File
@@ -7,7 +7,7 @@
"generator_options": "./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles '-YALAND10:Land area' '-L{\"file\": \"tests/join-population/tabblock_06001420.json\", \"description\": \"population\"}'; ./tile-join --quiet --force '--tile-stats-sample-values-limit=1' -o tests/join-population/joined-tile-stats-sample-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles",
"json": "{\"vector_layers\":[{\"id\":\"tabblock_06001420\",\"description\":\"population\",\"minzoom\":0,\"maxzoom\":12,\"fields\":{\"ALAND10\":\"Land area\",\"AWATER10\":\"Number\",\"BLOCKCE10\":\"String\",\"COUNTYFP10\":\"String\",\"FUNCSTAT10\":\"String\",\"INTPTLAT10\":\"String\",\"INTPTLON10\":\"String\",\"MTFCC10\":\"String\",\"NAME10\":\"String\",\"STATEFP10\":\"String\",\"TRACTCE10\":\"String\",\"UACE10\":\"String\",\"UATYP10\":\"String\",\"UR10\":\"String\",\"population\":\"String\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"tabblock_06001420\",\"count\":1734,\"geometry\":\"Polygon\",\"attributeCount\":15,\"attributes\":[{\"attribute\":\"ALAND10\",\"count\":1,\"type\":\"number\",\"values\":[0],\"min\":0,\"max\":542505},{\"attribute\":\"AWATER10\",\"count\":1,\"type\":\"number\",\"values\":[0],\"min\":0,\"max\":1632801},{\"attribute\":\"BLOCKCE10\",\"count\":1,\"type\":\"string\",\"values\":[\"1000\"]},{\"attribute\":\"COUNTYFP10\",\"count\":1,\"type\":\"string\",\"values\":[\"001\"]},{\"attribute\":\"FUNCSTAT10\",\"count\":1,\"type\":\"string\",\"values\":[\"S\"]},{\"attribute\":\"INTPTLAT10\",\"count\":1,\"type\":\"string\",\"values\":[\"+37.882724\"]},{\"attribute\":\"INTPTLON10\",\"count\":1,\"type\":\"string\",\"values\":[\"-122.282320\"]},{\"attribute\":\"MTFCC10\",\"count\":1,\"type\":\"string\",\"values\":[\"G5040\"]},{\"attribute\":\"NAME10\",\"count\":1,\"type\":\"string\",\"values\":[\"Block 1000\"]},{\"attribute\":\"STATEFP10\",\"count\":1,\"type\":\"string\",\"values\":[\"06\"]},{\"attribute\":\"TRACTCE10\",\"count\":1,\"type\":\"string\",\"values\":[\"420100\"]},{\"attribute\":\"UACE10\",\"count\":1,\"type\":\"string\",\"values\":[\"78904\"]},{\"attribute\":\"UATYP10\",\"count\":1,\"type\":\"string\",\"values\":[\"U\"]},{\"attribute\":\"UR10\",\"count\":1,\"type\":\"string\",\"values\":[\"R\"]},{\"attribute\":\"population\",\"count\":1,\"type\":\"string\",\"values\":[\"\"]}]}]}}",
"maxzoom": "12",
"minzoom": "0",
"minzoom": "3",
"name": "tests/join-population/tabblock_06001420.mbtiles",
"strategies": "[{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":197},{\"tiny_polygons\":195},{\"tiny_polygons\":189},{\"tiny_polygons\":97},{\"tiny_polygons\":34},{\"tiny_polygons\":24}]",
"type": "overlay",
tests/join-population/joined-tile-stats-values-limit.mbtiles.json
+1 -1
View File
@@ -7,7 +7,7 @@
"generator_options": "./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles '-YALAND10:Land area' '-L{\"file\": \"tests/join-population/tabblock_06001420.json\", \"description\": \"population\"}'; ./tile-join --quiet --force '--tile-stats-values-limit=1' -o tests/join-population/joined-tile-stats-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles",
"json": "{\"vector_layers\":[{\"id\":\"tabblock_06001420\",\"description\":\"population\",\"minzoom\":0,\"maxzoom\":12,\"fields\":{\"ALAND10\":\"Land area\",\"AWATER10\":\"Number\",\"BLOCKCE10\":\"String\",\"COUNTYFP10\":\"String\",\"FUNCSTAT10\":\"String\",\"INTPTLAT10\":\"String\",\"INTPTLON10\":\"String\",\"MTFCC10\":\"String\",\"NAME10\":\"String\",\"STATEFP10\":\"String\",\"TRACTCE10\":\"String\",\"UACE10\":\"String\",\"UATYP10\":\"String\",\"UR10\":\"String\",\"population\":\"Mixed\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"tabblock_06001420\",\"count\":1734,\"geometry\":\"Polygon\",\"attributeCount\":15,\"attributes\":[{\"attribute\":\"ALAND10\",\"count\":257,\"type\":\"number\",\"values\":[0],\"min\":0,\"max\":542505}]}]}}",
"maxzoom": "12",
"minzoom": "0",
"minzoom": "3",
"name": "tests/join-population/tabblock_06001420.mbtiles",
"strategies": "[{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":197},{\"tiny_polygons\":195},{\"tiny_polygons\":189},{\"tiny_polygons\":97},{\"tiny_polygons\":34},{\"tiny_polygons\":24}]",
"type": "overlay",
tests/join-population/joined.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/just-macarthur.mbtiles.json
+3 -3
View File
@@ -2,13 +2,13 @@
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"attribution": "macarthur's attribution",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"center": "-122.299805,37.892187,12",
"center": "-122.299805,37.892187,11",
"description": "macarthur description",
"format": "pbf",
"generator_options": "./tippecanoe -q -f -Z5 -z10 -o tests/join-population/macarthur.mbtiles -l macarthur tests/join-population/macarthur.json; ./tippecanoe -q -f -d10 -D10 -Z9 -z11 -o tests/join-population/macarthur2.mbtiles -l macarthur tests/join-population/macarthur2.json; ./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles '-YALAND10:Land area' '-L{\"file\": \"tests/join-population/tabblock_06001420.json\", \"description\": \"population\"}'; ./tile-join -q -f -o tests/join-population/merged.mbtiles tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles; ./tile-join -q -f -l macarthur -n 'macarthur name' -N 'macarthur description' -A 'macarthur'\"'\"'s attribution' -o tests/join-population/just-macarthur.mbtiles tests/join-population/merged.mbtiles",
"json": "{\"vector_layers\":[{\"id\":\"macarthur\",\"description\":\"\",\"minzoom\":5,\"maxzoom\":11,\"fields\":{\"FULLNAME\":\"String\",\"LINEARID\":\"String\",\"MTFCC\":\"String\",\"RTTYP\":\"String\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"macarthur\",\"count\":171,\"geometry\":\"LineString\",\"attributeCount\":4,\"attributes\":[{\"attribute\":\"FULLNAME\",\"count\":5,\"type\":\"string\",\"values\":[\"Macarthur\",\"Macarthur Blvd\",\"Macarthur Fwy\",\"W Macarthur\",\"W Macarthur Blvd\"]},{\"attribute\":\"LINEARID\",\"count\":42,\"type\":\"string\",\"values\":[\"1102155930810\",\"1102156217102\",\"1102156241736\",\"1102156248968\",\"1102156510290\",\"1102157651658\",\"1102406970092\",\"1102406970093\",\"1102406970094\",\"1102406970095\",\"1102407366406\",\"1102638069562\",\"1102638078801\",\"1102654601627\",\"1102654601663\",\"1102654602215\",\"1102954189105\",\"1102954918511\",\"1103690383700\",\"1103690474249\",\"1103690474250\",\"1103690483026\",\"1103690483032\",\"1103717593123\",\"1104469713187\",\"1104469713198\",\"1104474748623\",\"1104475134288\",\"1104475134436\",\"1104485605278\",\"1104485645649\",\"1104485773833\",\"1104486090991\",\"1104486392881\",\"1105089436004\",\"1105089465114\",\"1105089465116\",\"1105281275434\",\"1105281275687\",\"1105281275688\",\"1105281275689\",\"1105281275692\"]},{\"attribute\":\"MTFCC\",\"count\":2,\"type\":\"string\",\"values\":[\"S1100\",\"S1400\"]},{\"attribute\":\"RTTYP\",\"count\":1,\"type\":\"string\",\"values\":[\"M\"]}]}]}}",
"maxzoom": "12",
"minzoom": "0",
"maxzoom": "11",
"minzoom": "5",
"name": "macarthur name",
"strategies": "[{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":199},{\"tiny_polygons\":197},{\"tiny_polygons\":195},{\"tiny_polygons\":189},{\"tiny_polygons\":97},{\"tiny_polygons\":34},{\"tiny_polygons\":24}]",
"type": "overlay",
tests/join-population/merged-folder.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/merged.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/no-macarthur.mbtiles.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/join-population/raw-merged-folder.json
+1 -1
View File
File diff suppressed because one or more lines are too long
tests/wraparound2/in.json
+1
View File
File diff suppressed because one or more lines are too long
tests/wraparound2/out/-z0_--detect-longitude-wraparound.json
+20
View File
File diff suppressed because one or more lines are too long
tile-join.cpp
+24 -6
View File
@@ -434,7 +434,8 @@ struct tileset_reader {
// for overzooming
int maxzoom_so_far = -1;
std::vector<std::pair<unsigned, unsigned>> tiles_at_maxzoom_so_far;
std::vector<std::pair<unsigned, unsigned>> overzoomed_tiles;
std::vector<std::pair<unsigned, unsigned>> overzoomed_tiles; // tiles at `zoom`
std::vector<std::pair<unsigned, unsigned>> next_overzoomed_tiles; // tiles at `zoom + 1`
bool overzoom_consumed_at_this_zoom = false;
// parent tile cache
@@ -642,12 +643,24 @@ struct tileset_reader {
long long scale = (1LL << zoom) / (1LL << maxzoom_so_far);
for (auto const &xy : tiles_at_maxzoom_so_far) {
for (long long xx = 0; xx < scale; xx++) {
for (long long yy = 0; yy < scale; yy++) {
overzoomed_tiles.push_back(std::pair<unsigned, unsigned>(xy.first * scale + xx, xy.second * scale + yy));
// If this is the first overzoomed level, we don't know yet
// which tiles will be useful, so spell out all 4 child tiles
// from each parent tile.
//
// If it is further overzoomed than that, we have a list of
// which child tiles will have features in them, so use that.
if (zoom == maxzoom_so_far + 1) {
for (auto const &xy : tiles_at_maxzoom_so_far) {
for (long long xx = 0; xx < scale; xx++) {
for (long long yy = 0; yy < scale; yy++) {
overzoomed_tiles.push_back(std::pair<unsigned, unsigned>(xy.first * scale + xx, xy.second * scale + yy));
}
}
}
} else {
overzoomed_tiles = std::move(next_overzoomed_tiles);
next_overzoomed_tiles.clear();
}
std::sort(overzoomed_tiles.begin(), overzoomed_tiles.end(), tilecmp);
@@ -769,7 +782,7 @@ struct tileset_reader {
}
if (source.layers.size() != 0) {
std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false);
std::string ret = overzoom(source, parent_tile.z, parent_tile.x, parent_tile.y, tile.z, tile.x, tile.y, -1, buffer, std::set<std::string>(), false, &next_overzoomed_tiles);
return ret;
}
@@ -1574,6 +1587,11 @@ int main(int argc, char **argv) {
}
generator_options.append(commandline);
// don't trust the source metadata maxzooms;
// claim the zooms that were actually written
st.maxzoom = INT_MIN;
st.minzoom = INT_MAX;
for (auto &l : layermap) {
if (l.second.minzoom < st.minzoom) {
st.minzoom = l.second.minzoom;
version.hpp
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define VERSION "v2.35.0"
#define VERSION "v2.37.1"
#endif