diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd1024b..78bf5b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 2.74.0 + +* Add the option to join attributes from a sqlite database in tile-join +* Improve tile-join's tileset bounding box calculations + # 2.73.0 * Correctly clip features down to nothing when the clip region doesn't intersect the tile at all diff --git a/Makefile b/Makefile index 2f522d00..fc40467b 100644 --- a/Makefile +++ b/Makefile @@ -578,6 +578,20 @@ join-test: tippecanoe tippecanoe-decode tile-join ./tippecanoe-decode -x generator tests/ne_110m_ocean/join/joined.mbtiles > tests/ne_110m_ocean/join/joined.mbtiles.json.check cmp tests/ne_110m_ocean/join/joined.mbtiles.json.check tests/ne_110m_ocean/join/joined.mbtiles.json rm -f tests/ne_110m_ocean/join/ocean.mbtiles tests/ne_110m_ocean/join/countries.mbtiles tests/ne_110m_ocean/join/joined.mbtiles tests/ne_110m_ocean/join/joined.mbtiles.json.check + # + # Test sql join + # + ./tile-join -i -f -o tests/join-sql/countries.pmtiles --join-sqlite tests/join-sql/countries.gpkg --join-table countries --join-tile-attribute ne10-admin0:name_en --join-table-expression 'lower(country)' tests/join-sql/bboxes.pmtiles + ./tippecanoe-decode -x generator tests/join-sql/countries.pmtiles > tests/join-sql/countries.pmtiles.json.check + cmp tests/join-sql/countries.pmtiles.json.check tests/join-sql/countries.pmtiles.json + rm -f tests/join-sql/countries.pmtiles tests/join-sql/countries.pmtiles.json.check + # + # Test sql join with limit + # + ./tile-join --join-count-limit 3 -i -f -o tests/join-sql/countries-limit3.pmtiles --join-sqlite tests/join-sql/countries.gpkg --join-table countries --join-tile-attribute ne10-admin0:name_en --join-table-expression 'lower(country)' tests/join-sql/bboxes.pmtiles + ./tippecanoe-decode -x generator tests/join-sql/countries-limit3.pmtiles > tests/join-sql/countries-limit3.pmtiles.json.check + cmp tests/join-sql/countries-limit3.pmtiles.json.check tests/join-sql/countries-limit3.pmtiles.json + rm -f tests/join-sql/countries-limit3.pmtiles tests/join-sql/countries-limit3.pmtiles.json.check accumulate-test: # there are 144 features with POP1950 in the original dataset diff --git a/tests/feature-filter/out/filtered.json.standard b/tests/feature-filter/out/filtered.json.standard index 73fb672d..c391431c 100644 --- a/tests/feature-filter/out/filtered.json.standard +++ b/tests/feature-filter/out/filtered.json.standard @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129", -"bounds": "-180.000000,-85.051129,180.000000,85.051129", +"antimeridian_adjusted_bounds": "-100.019531,0.000000,0.966797,0.966751", +"bounds": "-100.019531,0.000000,0.966797,0.966751", "center": "0.000000,0.000000,0", "description": "tests/feature-filter/out/all.mbtiles", "format": "pbf", diff --git a/tests/feature-filter/out/places-filter.mbtiles.json.standard b/tests/feature-filter/out/places-filter.mbtiles.json.standard index 9936c097..ab8b5a1f 100644 --- a/tests/feature-filter/out/places-filter.mbtiles.json.standard +++ b/tests/feature-filter/out/places-filter.mbtiles.json.standard @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-175.781250,-42.032974,180.000000,64.168107", -"bounds": "-175.781250,-42.032974,180.000000,64.168107", +"antimeridian_adjusted_bounds": "-175.220947,-41.302571,179.217224,64.168107", +"bounds": "-175.220947,-41.302571,179.217224,64.168107", "center": "-62.578125,17.307462,8", "description": "tests/feature-filter/out/places.mbtiles", "format": "pbf", diff --git a/tests/join-population/concat.mbtiles.json b/tests/join-population/concat.mbtiles.json index 28a2a402..156596d7 100644 --- a/tests/join-population/concat.mbtiles.json +++ b/tests/join-population/concat.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129", -"bounds": "-180.000000,-85.051129,180.000000,85.051129", +"antimeridian_adjusted_bounds": "-122.255859,37.718590,-122.080078,37.857507", +"bounds": "-122.255859,37.718590,-122.080078,37.857507", "center": "-122.104097,37.695438,0", "description": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "format": "pbf", diff --git a/tests/join-population/joined-i.mbtiles.json b/tests/join-population/joined-i.mbtiles.json index 35a15fd7..6d8dd763 100644 --- a/tests/join-population/joined-i.mbtiles.json +++ b/tests/join-population/joined-i.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.309418,37.881357,-122.280579,37.900865", +"bounds": "-122.309418,37.881357,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-no-tile-stats.mbtiles.json b/tests/join-population/joined-no-tile-stats.mbtiles.json index 6f7cde11..4e5b7ec0 100644 --- a/tests/join-population/joined-no-tile-stats.mbtiles.json +++ b/tests/join-population/joined-no-tile-stats.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-null.mbtiles.json b/tests/join-population/joined-null.mbtiles.json index 8339f194..c8a71676 100644 --- a/tests/join-population/joined-null.mbtiles.json +++ b/tests/join-population/joined-null.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json b/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json index 37b5326a..c6f6d52c 100644 --- a/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json b/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json index 4b217de0..b3ce3787 100644 --- a/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined-tile-stats-values-limit.mbtiles.json b/tests/join-population/joined-tile-stats-values-limit.mbtiles.json index 9fa16025..7999ec3c 100644 --- a/tests/join-population/joined-tile-stats-values-limit.mbtiles.json +++ b/tests/join-population/joined-tile-stats-values-limit.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/joined.mbtiles.json b/tests/join-population/joined.mbtiles.json index 16ad55ee..4d416618 100644 --- a/tests/join-population/joined.mbtiles.json +++ b/tests/join-population/joined.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868", -"bounds": "-122.343750,37.857507,-122.255859,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/just-macarthur.mbtiles.json b/tests/join-population/just-macarthur.mbtiles.json index 97b1b7e1..c1a48b33 100644 --- a/tests/join-population/just-macarthur.mbtiles.json +++ b/tests/join-population/just-macarthur.mbtiles.json @@ -1,7 +1,7 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", "attribution": "macarthur's attribution", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.299805,37.892187,11", "description": "macarthur description", "format": "pbf", diff --git a/tests/join-population/macarthur-6-9-exclude.mbtiles.json b/tests/join-population/macarthur-6-9-exclude.mbtiles.json index 0e191929..f079ea24 100644 --- a/tests/join-population/macarthur-6-9-exclude.mbtiles.json +++ b/tests/join-population/macarthur-6-9-exclude.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107", +"bounds": "-122.294655,37.695231,-122.103424,37.833107", "center": "-122.167969,37.833010,9", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/join-population/macarthur-6-9.mbtiles.json b/tests/join-population/macarthur-6-9.mbtiles.json index 3f79712d..0452f638 100644 --- a/tests/join-population/macarthur-6-9.mbtiles.json +++ b/tests/join-population/macarthur-6-9.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107", +"bounds": "-122.294655,37.695231,-122.103424,37.833107", "center": "-122.167969,37.833010,9", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/join-population/merged-folder.mbtiles.json b/tests/join-population/merged-folder.mbtiles.json index 86337ba1..3e5f1cd4 100644 --- a/tests/join-population/merged-folder.mbtiles.json +++ b/tests/join-population/merged-folder.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420-folder", "format": "pbf", diff --git a/tests/join-population/merged.mbtiles.json b/tests/join-population/merged.mbtiles.json index e2106996..17d42609 100644 --- a/tests/join-population/merged.mbtiles.json +++ b/tests/join-population/merged.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/no-macarthur.mbtiles.json b/tests/join-population/no-macarthur.mbtiles.json index 80a22b9e..5d33240a 100644 --- a/tests/join-population/no-macarthur.mbtiles.json +++ b/tests/join-population/no-macarthur.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865", +"bounds": "-122.343750,37.874853,-122.280579,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/raw-merged-folder.json b/tests/join-population/raw-merged-folder.json index 2c7607ce..a61b2cee 100644 --- a/tests/join-population/raw-merged-folder.json +++ b/tests/join-population/raw-merged-folder.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868", -"bounds": "-122.343750,37.695438,-122.104097,37.926868", +"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865", +"bounds": "-122.343750,37.694688,-122.103424,37.900865", "center": "-122.299805,37.892187,12", "description": "tests/join-population/tabblock_06001420.mbtiles", "format": "pbf", diff --git a/tests/join-population/renamed.mbtiles.json b/tests/join-population/renamed.mbtiles.json index dc74bcfb..58bcc153 100644 --- a/tests/join-population/renamed.mbtiles.json +++ b/tests/join-population/renamed.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.167969,37.828608,10", "description": "tests/join-population/macarthur2.mbtiles", "format": "pbf", diff --git a/tests/join-population/windows.mbtiles.json b/tests/join-population/windows.mbtiles.json index ef2dcc14..570b1710 100644 --- a/tests/join-population/windows.mbtiles.json +++ b/tests/join-population/windows.mbtiles.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163", -"bounds": "-122.343750,37.439974,-121.992188,37.996163", +"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649", +"bounds": "-122.294655,37.694688,-122.103424,37.833649", "center": "-122.167969,37.833010,10", "description": "tests/join-population/macarthur.mbtiles", "format": "pbf", diff --git a/tests/join-sql/bboxes.pmtiles b/tests/join-sql/bboxes.pmtiles new file mode 100644 index 00000000..12c79e0f Binary files /dev/null and b/tests/join-sql/bboxes.pmtiles differ diff --git a/tests/join-sql/countries-limit3.pmtiles.json b/tests/join-sql/countries-limit3.pmtiles.json new file mode 100644 index 00000000..86f396fc --- /dev/null +++ b/tests/join-sql/countries-limit3.pmtiles.json @@ -0,0 +1,33 @@ +{ "type": "FeatureCollection", "properties": { +"antimeridian_adjusted_bounds": "-63.146840,-21.370782,55.854503,55.065334", +"bounds": "-63.146840,-21.370782,55.854503,55.065334", +"center": "0.000000,0.000000,0", +"description": "/tmp/tmp588djzw6/bboxes.pmtiles", +"format": "pbf", +"generator_options": "tippecanoe '--tile-stats-values-limit=1' -o /tmp/tmp588djzw6/bboxes.pmtiles -l parsed-bboxes -U1 '--smallest-maximum-zoom-guess=7' -Bg --drop-fraction-as-needed -P --hilbert --generate-variable-depth-tile-pyramid -rp -b0 '--extend-zooms-if-still-dropping-maximum=3' '--extra-detail=30' -D10 '--tiny-polygon-size=1' --no-tiny-polygon-reduction-at-maximum-zoom '--simplification=1' '--simplification-at-maximum-zoom=0.25' --detect-longitude-wraparound --preserve-input-order '--maximum-tile-bytes=2621440' '--maximum-string-attribute-length=65536' --preserve-point-density-threshold 64 --set-attribute '{\"felt:cluster_size\": 1}' --accumulate-attribute '{\"felt:cluster_size\": \"sum\"}' --preserve-multiplier-density-threshold 512 '--accumulate-numeric-attributes=felt' /tmp/tmpg9jf8evz/parsed.geojsonseq.gz; ./tile-join --join-count-limit 3 -i -f -o tests/join-sql/countries-limit3.pmtiles --join-sqlite tests/join-sql/countries.gpkg --join-table countries --join-tile-attribute ne10-admin0:name_en --join-table-expression 'lower(country)' tests/join-sql/bboxes.pmtiles", +"json": "{\"vector_layers\":[{\"id\":\"parsed-bboxes\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{\"another\":\"String\",\"country\":\"String\",\"felt:cluster_size\":\"Number\",\"fid\":\"Number\",\"ne10-admin0:iso_a2_eh\":\"String\",\"ne10-admin0:iso_a3_eh\":\"String\",\"ne10-admin0:name_en\":\"String\",\"something\":\"String\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"parsed-bboxes\",\"count\":7,\"geometry\":\"Polygon\",\"attributeCount\":8,\"attributes\":[{\"attribute\":\"another\",\"count\":6,\"type\":\"string\",\"values\":[\"bar\",\"deux\",\"hey\",\"two\",\"why\",\"yes\"]},{\"attribute\":\"country\",\"count\":4,\"type\":\"string\",\"values\":[\"France\",\"Germany\",\"Italy\",\"saint-martin\"]},{\"attribute\":\"felt:cluster_size\",\"count\":1,\"type\":\"number\",\"values\":[1],\"min\":1,\"max\":1},{\"attribute\":\"fid\",\"count\":6,\"type\":\"number\",\"values\":[1,2,3,4,5,7],\"min\":1,\"max\":7},{\"attribute\":\"ne10-admin0:iso_a2_eh\",\"count\":5,\"type\":\"string\",\"values\":[\"de\",\"fr\",\"it\",\"mf\",\"sx\"]},{\"attribute\":\"ne10-admin0:iso_a3_eh\",\"count\":5,\"type\":\"string\",\"values\":[\"deu\",\"fra\",\"ita\",\"maf\",\"sxm\"]},{\"attribute\":\"ne10-admin0:name_en\",\"count\":4,\"type\":\"string\",\"values\":[\"france\",\"germany\",\"italy\",\"saint-martin\"]},{\"attribute\":\"something\",\"count\":6,\"type\":\"string\",\"values\":[\"blah\",\"foo\",\"hey\",\"one\",\"un\",\"what\"]}]}]}}", +"maxzoom": "0", +"minzoom": "0", +"name": "/tmp/tmp588djzw6/bboxes.pmtiles", +"strategies": "[{\"truncated_zooms\":1}]", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "parsed-bboxes", "version": 2, "extent": 1073741824 }, "features": [ +{ "type": "Feature", "id": 322, "properties": { "ne10-admin0:name_en": "germany", "ne10-admin0:iso_a2_eh": "de", "ne10-admin0:iso_a3_eh": "deu", "felt:cluster_size": 1, "another": "yes", "country": "Germany", "fid": 1, "something": "blah" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 15.022060, 47.271121 ], [ 5.852490, 47.271121 ], [ 5.852490, 55.065334 ], [ 15.022060, 55.065334 ], [ 15.022060, 47.271121 ] ] ] } } +, +{ "type": "Feature", "id": 435, "properties": { "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1, "another": "why", "country": "France", "fid": 2, "something": "what" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } +, +{ "type": "Feature", "id": 435, "properties": { "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1, "another": "two", "country": "France", "fid": 4, "something": "one" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } +, +{ "type": "Feature", "id": 435, "properties": { "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1, "another": "deux", "country": "France", "fid": 5, "something": "un" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } +, +{ "type": "Feature", "id": 650, "properties": { "ne10-admin0:name_en": "italy", "ne10-admin0:iso_a2_eh": "it", "ne10-admin0:iso_a3_eh": "ita", "felt:cluster_size": 1, "another": "bar", "country": "Italy", "fid": 3, "something": "foo" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 18.517426, 35.489244 ], [ 6.602728, 35.489244 ], [ 6.602728, 47.085215 ], [ 18.517426, 47.085215 ], [ 18.517426, 35.489244 ] ] ] } } +, +{ "type": "Feature", "id": 1546, "properties": { "ne10-admin0:name_en": "saint-martin", "ne10-admin0:iso_a2_eh": "mf", "ne10-admin0:iso_a3_eh": "maf", "felt:cluster_size": 1, "another": "hey", "country": "saint-martin", "fid": 7, "something": "hey" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -63.010731, 18.033391 ], [ -63.146840, 18.033391 ], [ -63.146840, 18.122138 ], [ -63.010731, 18.122138 ], [ -63.010731, 18.033391 ] ] ] } } +, +{ "type": "Feature", "id": 1611, "properties": { "ne10-admin0:name_en": "saint-martin", "ne10-admin0:iso_a2_eh": "sx", "ne10-admin0:iso_a3_eh": "sxm", "felt:cluster_size": 1, "another": "hey", "country": "saint-martin", "fid": 7, "something": "hey" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -63.017568, 18.019110 ], [ -63.118886, 18.019110 ], [ -63.118886, 18.062120 ], [ -63.017568, 18.062120 ], [ -63.017568, 18.019110 ] ] ] } } +] } +] } +] } diff --git a/tests/join-sql/countries.gpkg b/tests/join-sql/countries.gpkg new file mode 100644 index 00000000..80a36ceb Binary files /dev/null and b/tests/join-sql/countries.gpkg differ diff --git a/tests/join-sql/countries.pmtiles.json b/tests/join-sql/countries.pmtiles.json new file mode 100644 index 00000000..99884911 --- /dev/null +++ b/tests/join-sql/countries.pmtiles.json @@ -0,0 +1,29 @@ +{ "type": "FeatureCollection", "properties": { +"antimeridian_adjusted_bounds": "-63.146840,-21.370782,55.854503,55.065334", +"bounds": "-63.146840,-21.370782,55.854503,55.065334", +"center": "0.000000,0.000000,0", +"description": "/tmp/tmp588djzw6/bboxes.pmtiles", +"format": "pbf", +"generator_options": "tippecanoe '--tile-stats-values-limit=1' -o /tmp/tmp588djzw6/bboxes.pmtiles -l parsed-bboxes -U1 '--smallest-maximum-zoom-guess=7' -Bg --drop-fraction-as-needed -P --hilbert --generate-variable-depth-tile-pyramid -rp -b0 '--extend-zooms-if-still-dropping-maximum=3' '--extra-detail=30' -D10 '--tiny-polygon-size=1' --no-tiny-polygon-reduction-at-maximum-zoom '--simplification=1' '--simplification-at-maximum-zoom=0.25' --detect-longitude-wraparound --preserve-input-order '--maximum-tile-bytes=2621440' '--maximum-string-attribute-length=65536' --preserve-point-density-threshold 64 --set-attribute '{\"felt:cluster_size\": 1}' --accumulate-attribute '{\"felt:cluster_size\": \"sum\"}' --preserve-multiplier-density-threshold 512 '--accumulate-numeric-attributes=felt' /tmp/tmpg9jf8evz/parsed.geojsonseq.gz; ./tile-join -i -f -o tests/join-sql/countries.pmtiles --join-sqlite tests/join-sql/countries.gpkg --join-table countries --join-tile-attribute ne10-admin0:name_en --join-table-expression 'lower(country)' tests/join-sql/bboxes.pmtiles", +"json": "{\"vector_layers\":[{\"id\":\"parsed-bboxes\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{\"another\":\"String\",\"country\":\"String\",\"felt:cluster_size\":\"Number\",\"fid\":\"Number\",\"ne10-admin0:iso_a2_eh\":\"String\",\"ne10-admin0:iso_a3_eh\":\"String\",\"ne10-admin0:name_en\":\"String\",\"something\":\"String\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"parsed-bboxes\",\"count\":5,\"geometry\":\"Polygon\",\"attributeCount\":8,\"attributes\":[{\"attribute\":\"another\",\"count\":4,\"type\":\"string\",\"values\":[\"bar\",\"hey\",\"why\",\"yes\"]},{\"attribute\":\"country\",\"count\":4,\"type\":\"string\",\"values\":[\"France\",\"Germany\",\"Italy\",\"saint-martin\"]},{\"attribute\":\"felt:cluster_size\",\"count\":1,\"type\":\"number\",\"values\":[1],\"min\":1,\"max\":1},{\"attribute\":\"fid\",\"count\":4,\"type\":\"number\",\"values\":[1,2,3,7],\"min\":1,\"max\":7},{\"attribute\":\"ne10-admin0:iso_a2_eh\",\"count\":5,\"type\":\"string\",\"values\":[\"de\",\"fr\",\"it\",\"mf\",\"sx\"]},{\"attribute\":\"ne10-admin0:iso_a3_eh\",\"count\":5,\"type\":\"string\",\"values\":[\"deu\",\"fra\",\"ita\",\"maf\",\"sxm\"]},{\"attribute\":\"ne10-admin0:name_en\",\"count\":4,\"type\":\"string\",\"values\":[\"france\",\"germany\",\"italy\",\"saint-martin\"]},{\"attribute\":\"something\",\"count\":4,\"type\":\"string\",\"values\":[\"blah\",\"foo\",\"hey\",\"what\"]}]}]}}", +"maxzoom": "0", +"minzoom": "0", +"name": "/tmp/tmp588djzw6/bboxes.pmtiles", +"strategies": "[{\"truncated_zooms\":1}]", +"type": "overlay", +"version": "2" +}, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "parsed-bboxes", "version": 2, "extent": 1073741824 }, "features": [ +{ "type": "Feature", "id": 322, "properties": { "ne10-admin0:name_en": "germany", "ne10-admin0:iso_a2_eh": "de", "ne10-admin0:iso_a3_eh": "deu", "felt:cluster_size": 1, "another": "yes", "country": "Germany", "fid": 1, "something": "blah" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 15.022060, 47.271121 ], [ 5.852490, 47.271121 ], [ 5.852490, 55.065334 ], [ 15.022060, 55.065334 ], [ 15.022060, 47.271121 ] ] ] } } +, +{ "type": "Feature", "id": 435, "properties": { "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1, "another": "why", "country": "France", "fid": 2, "something": "what" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } +, +{ "type": "Feature", "id": 650, "properties": { "ne10-admin0:name_en": "italy", "ne10-admin0:iso_a2_eh": "it", "ne10-admin0:iso_a3_eh": "ita", "felt:cluster_size": 1, "another": "bar", "country": "Italy", "fid": 3, "something": "foo" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 18.517426, 35.489244 ], [ 6.602728, 35.489244 ], [ 6.602728, 47.085215 ], [ 18.517426, 47.085215 ], [ 18.517426, 35.489244 ] ] ] } } +, +{ "type": "Feature", "id": 1546, "properties": { "ne10-admin0:name_en": "saint-martin", "ne10-admin0:iso_a2_eh": "mf", "ne10-admin0:iso_a3_eh": "maf", "felt:cluster_size": 1, "another": "hey", "country": "saint-martin", "fid": 7, "something": "hey" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -63.010731, 18.033391 ], [ -63.146840, 18.033391 ], [ -63.146840, 18.122138 ], [ -63.010731, 18.122138 ], [ -63.010731, 18.033391 ] ] ] } } +, +{ "type": "Feature", "id": 1611, "properties": { "ne10-admin0:name_en": "saint-martin", "ne10-admin0:iso_a2_eh": "sx", "ne10-admin0:iso_a3_eh": "sxm", "felt:cluster_size": 1, "another": "hey", "country": "saint-martin", "fid": 7, "something": "hey" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -63.017568, 18.019110 ], [ -63.118886, 18.019110 ], [ -63.118886, 18.062120 ], [ -63.017568, 18.062120 ], [ -63.017568, 18.019110 ] ] ] } } +] } +] } +] } diff --git a/tests/ne_110m_ocean/join/joined.mbtiles.json b/tests/ne_110m_ocean/join/joined.mbtiles.json index 08148a95..4c2c3c80 100644 --- a/tests/ne_110m_ocean/join/joined.mbtiles.json +++ b/tests/ne_110m_ocean/join/joined.mbtiles.json @@ -1,5 +1,5 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129", +"antimeridian_adjusted_bounds": "0.000000,-85.051129,359.912109,85.051129", "bounds": "-180.000000,-85.051129,180.000000,85.051129", "center": "-45.000000,33.256630,4", "description": "tests/ne_110m_ocean/join/ocean.mbtiles", diff --git a/tests/raw-tiles/raw-tiles-z67-join.json b/tests/raw-tiles/raw-tiles-z67-join.json index d82e92ee..8582128c 100644 --- a/tests/raw-tiles/raw-tiles-z67-join.json +++ b/tests/raw-tiles/raw-tiles-z67-join.json @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "properties": { -"antimeridian_adjusted_bounds": "-123.750000,45.089036,-120.937500,47.040182", -"bounds": "-123.750000,45.089036,-120.937500,47.040182", +"antimeridian_adjusted_bounds": "-122.682953,45.512121,-122.654800,45.569832", +"bounds": "-122.682953,45.512121,-122.654800,45.569832", "center": "-122.662354,45.514045,7", "description": "tests/raw-tiles/raw-tiles", "format": "pbf", diff --git a/tile-join.cpp b/tile-join.cpp index ee888959..d9484a28 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -55,7 +55,13 @@ int maxzoom = 32; int minzoom = 0; std::map renames; bool exclude_all = false; +bool exclude_all_tile_attributes = false; std::vector unidecode_data; +std::string join_tile_attribute; +std::string join_table_expression; +std::string join_table; +size_t join_count_limit = 1; +std::string attribute_for_id; bool want_overzoom = false; int buffer = 5; @@ -73,7 +79,115 @@ struct stats { std::vector strategies{}; }; -void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) { +// list, per feature in the tile, +// of lists of features in the sqlite response, +// each of which is a mapping from keys to values +std::vector>> get_joined_rows(sqlite3 *db, const std::vector &join_keys) { + std::vector>> ret; + ret.resize(join_keys.size()); + + // double quotes for table and column identifiers + const char *s = sqlite3_mprintf("select %s, * from \"%w\" where %s in (", + join_table_expression.c_str(), join_table.c_str(), join_table_expression.c_str()); + std::string query = s; + sqlite3_free((void *) s); + + std::multimap key_to_row; + for (size_t i = 0; i < join_keys.size(); i++) { + const mvt_value &v = join_keys[i]; + + // single quotes for literals + if (v.type == mvt_string) { + s = sqlite3_mprintf("'%q'", v.c_str()); + query += s; + sqlite3_free((void *) s); + key_to_row.emplace(v.get_string_value(), i); + } else { + std::string stringified = v.toString(); + key_to_row.emplace(stringified, i); + query += stringified; + } + + if (i + 1 < join_keys.size()) { + query += ", "; + } + } + + // this doesn't add a LIMIT to the query because our limit + // is per tiled feature, not a limit on the entire query response. + query += ");"; + + sqlite3_stmt *stmt; + if (sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, NULL) != SQLITE_OK) { + fprintf(stderr, "sqlite3 query %s failed: %s\n", query.c_str(), sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + while (sqlite3_step(stmt) == SQLITE_ROW) { + int count = sqlite3_column_count(stmt); + std::map row; + + if (count > 0) { + // join key is 0th column of query + std::string key = (const char *) sqlite3_column_text(stmt, 0); + auto f = key_to_row.equal_range(key); + if (f.first == f.second) { + fprintf(stderr, "Unexpected join key: %s\n", key.c_str()); + continue; + } + + for (auto ff = f.first; ff != f.second; ++ff) { + if (ret[ff->second].size() < join_count_limit) { + for (int i = 1; i < count; i++) { + int type = sqlite3_column_type(stmt, i); + mvt_value v; + v.type = mvt_null; + + if (type == SQLITE_INTEGER || type == SQLITE_FLOAT) { + v = mvt_value(sqlite3_column_double(stmt, i)); + } else if (type == SQLITE_TEXT || type == SQLITE_BLOB) { + v.set_string_value((const char *) sqlite3_column_text(stmt, i)); + } + + const char *name = sqlite3_column_name(stmt, i); + row.emplace(name, v); + } + + ret[ff->second].push_back(row); + } + } + } + } + if (sqlite3_finalize(stmt) != SQLITE_OK) { + fprintf(stderr, "sqlite3 finalize failed: %s\n", sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + + return ret; +} + +struct arg { + std::map> inputs{}; + std::map outputs{}; + + std::map *layermap = NULL; + + std::vector *header = NULL; + std::map> *mapping = NULL; + sqlite3 *db = NULL; + std::set *exclude = NULL; + std::set *include = NULL; + std::set *keep_layers = NULL; + std::set *remove_layers = NULL; + int ifmatched = 0; + json_object *filter = NULL; + struct tileset_reader *readers = NULL; + + double minlat, minlon; + double maxlat, maxlon; + double minlon2, maxlon2; +}; + +void append_tile(std::string message, int z, unsigned x, unsigned y, std::map &layermap, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, std::set &keep_layers, std::set &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter, struct arg *a) { mvt_tile tile; int features_added = 0; bool was_compressed; @@ -139,96 +253,210 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>> joined; + if (db != NULL) { + // collect join keys for sql query - for (size_t f = 0; f < layer.features.size(); f++) { - mvt_feature feat = layer.features[f]; - std::set exclude_attributes; + std::vector join_keys; + join_keys.resize(layer.features.size()); - if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z, unidecode_data)) { - continue; - } + for (size_t f = 0; f < layer.features.size(); f++) { + mvt_feature &feat = layer.features[f]; + join_keys[f].type = mvt_no_such_key; - mvt_feature outfeature; - int matched = 0; - - if (feat.has_id) { - outfeature.has_id = true; - outfeature.id = feat.id; - } - - std::map> attributes; - std::vector key_order; - - for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { - const char *key = layer.keys[feat.tags[t]].c_str(); - mvt_value &val = layer.values[feat.tags[t + 1]]; - serial_val sv = mvt_value_to_serial_val(val); - - if (sv.type == mvt_null) { - continue; - } - - if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) { - attributes.insert(std::pair>(key, std::pair(val, sv))); - key_order.push_back(key); - } - - if (header.size() > 0 && strcmp(key, header[0].c_str()) == 0) { - std::map>::iterator ii = mapping.find(sv.s); - - if (ii != mapping.end()) { - std::vector fields = ii->second; - matched = 1; - - for (size_t i = 1; i < fields.size(); i++) { - std::string joinkey = header[i]; - std::string joinval = fields[i]; - int attr_type = mvt_string; - - if (joinval.size() > 0) { - if (joinval[0] == '"') { - joinval = csv_dequote(joinval); - } else if (is_number(joinval)) { - attr_type = mvt_double; - } - } else if (pe) { - attr_type = mvt_null; - } - - const char *sjoinkey = joinkey.c_str(); - - if (include.count(joinkey) || (!exclude_all && exclude.count(joinkey) == 0 && exclude_attributes.count(joinkey) == 0 && attr_type != mvt_null)) { - mvt_value outval; - if (attr_type == mvt_string) { - outval.type = mvt_string; - outval.set_string_value(joinval); - } else { - outval.type = mvt_double; - outval.numeric_value.double_value = atof(joinval.c_str()); - } - - auto fa = attributes.find(sjoinkey); - if (fa != attributes.end()) { - attributes.erase(fa); - } - - serial_val outsv; - outsv.type = outval.type; - outsv.s = joinval; - - // Convert from double to int if the joined attribute is an integer - outval = stringified_to_mvt_value(outval.type, joinval.c_str(), tile_stringpool); - - attributes.insert(std::pair>(joinkey, std::pair(outval, outsv))); - key_order.push_back(joinkey); - } - } + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + if (key == join_tile_attribute) { + const mvt_value &val = layer.values[feat.tags[t + 1]]; + join_keys[f] = val; + break; } } } - if (matched || !ifmatched) { + joined = get_joined_rows(db, join_keys); + } + + auto tilestats = layermap.find(layer.name); + + long long minx = LLONG_MAX; + long long miny = LLONG_MAX; + long long maxx = LLONG_MIN; + long long maxy = LLONG_MIN; + + long long minx2 = LLONG_MAX; + long long maxx2 = LLONG_MIN; + bool features_added_to_layer = false; + + for (size_t f = 0; f < layer.features.size(); f++) { + mvt_feature &feat = layer.features[f]; + + std::set exclude_attributes; + if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z, unidecode_data)) { + continue; + } + + struct match { + bool has_id = false; + unsigned long long id; + std::map> attributes; + std::vector key_order; + }; + + std::vector matches; + bool matched = false; + + // start filling out sql matches + + if (f < joined.size()) { + if (joined[f].size() > 0) { + matched = true; + } + + for (auto const &joined_feature : joined[f]) { + match m; + m.has_id = feat.has_id; + m.id = feat.id; + + if (!exclude_all_tile_attributes) { + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + serial_val sv = mvt_value_to_serial_val(val); + + if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) { + m.attributes.insert(std::pair>(key, std::pair(val, sv))); + m.key_order.push_back(key); + } + } + } + + for (auto const &kv : joined_feature) { + if (kv.first == attribute_for_id) { + m.has_id = true; + m.id = mvt_value_to_long_long(kv.second); + } else if (include.count(kv.first) || (!exclude_all && exclude.count(kv.first) == 0 && exclude_attributes.count(kv.first) == 0)) { + if (kv.second.type != mvt_null) { + m.attributes.insert(std::pair>(kv.first, std::pair(kv.second, mvt_value_to_serial_val(kv.second)))); + m.key_order.push_back(kv.first); + } + } + } + + matches.push_back(m); + } + } + + // look for csv matches and start filling them out + + if (!matched) { + match m; + m.id = feat.id; + m.has_id = feat.has_id; + // populate attributes and key_order as we look for matches, + // because apparently at some point i thought it was important + // to insert the joined attributes at the point in the sequence + // where the join key had been + + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + serial_val sv = mvt_value_to_serial_val(val); + + if (val.type == mvt_null) { + continue; + } + + if (!exclude_all_tile_attributes) { + if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) { + m.attributes.insert(std::pair>(key, std::pair(val, sv))); + m.key_order.push_back(key); + } + } + + if (header.size() > 0 && key == header[0]) { + std::map>::iterator ii = mapping.find(sv.s); + + if (ii != mapping.end()) { + std::vector fields = ii->second; + matched = true; + + for (size_t i = 1; i < fields.size(); i++) { + std::string joinkey = header[i]; + std::string joinval = fields[i]; + int attr_type = mvt_string; + + if (joinval.size() > 0) { + if (joinval[0] == '"') { + joinval = csv_dequote(joinval); + } else if (is_number(joinval)) { + attr_type = mvt_double; + } + } else if (pe) { + attr_type = mvt_null; + } + + const char *sjoinkey = joinkey.c_str(); + + if (include.count(joinkey) || (!exclude_all && exclude.count(joinkey) == 0 && exclude_attributes.count(joinkey) == 0 && attr_type != mvt_null)) { + mvt_value outval; + if (attr_type == mvt_string) { + outval.type = mvt_string; + outval.set_string_value(joinval); + } else { + outval.type = mvt_double; + outval.numeric_value.double_value = atof(joinval.c_str()); + } + + auto fa = m.attributes.find(sjoinkey); + if (fa != m.attributes.end()) { + m.attributes.erase(fa); + } + + serial_val outsv; + outsv.type = outval.type; + outsv.s = joinval; + + outval = stringified_to_mvt_value(outval.type, joinval.c_str(), tile_stringpool); + + m.attributes.insert(std::pair>(joinkey, std::pair(outval, outsv))); + m.key_order.push_back(joinkey); + } + } + } + } + } + + if (matched) { + matches.push_back(m); + } + } + + if (!matched && !ifmatched) { + // no matches, but they said to keep even unmatched tile features, + // so make one that is just the original feature + + match m; + m.id = feat.id; + m.has_id = feat.has_id; + + if (!exclude_all_tile_attributes) { + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + serial_val sv = mvt_value_to_serial_val(val); + + if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) { + m.attributes.insert(std::pair>(key, std::pair(val, sv))); + m.key_order.push_back(key); + } + } + } + + matches.push_back(m); + } + + for (auto &m : matches) { if (tilestats == layermap.end()) { layermap.insert(std::pair(layer.name, layermap_entry(layermap.size()))); tilestats = layermap.find(layer.name); @@ -236,14 +464,18 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.maxzoom = z; } - // To keep attributes in their original order instead of alphabetical - for (auto k : key_order) { - auto fa = attributes.find(k); + mvt_feature outfeature; + outfeature.id = m.id; + outfeature.has_id = m.has_id; - if (fa != attributes.end()) { + // To keep attributes in their original order instead of alphabetical + for (auto k : m.key_order) { + auto fa = m.attributes.find(k); + + if (fa != m.attributes.end()) { outlayer.tag(outfeature, k, fa->second.first); add_to_tilestats(tilestats->second.tilestats, k, fa->second.second); - attributes.erase(fa); + m.attributes.erase(fa); } } @@ -257,7 +489,37 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.minzoom) { @@ -276,6 +538,24 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapminlat = std::min(a->minlat, std::min(lat1, lat2)); + a->minlon = std::min(a->minlon, std::min(lon1, lon2)); + a->maxlat = std::max(a->maxlat, std::max(lat1, lat2)); + a->maxlon = std::max(a->maxlon, std::max(lon1, lon2)); + + tile2lonlat(minx2, maxy, 32, &lon1, &lat1); + tile2lonlat(maxx2, miny, 32, &lon2, &lat2); + + a->minlon2 = std::min(a->minlon2, std::min(lon1, lon2)); + a->maxlon2 = std::max(a->maxlon2, std::max(lon1, lon2)); + } } if (features_added == 0) { @@ -283,22 +563,6 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map b) { - return a; - } else { - return b; - } -} - struct tilecmp { bool operator()(std::pair const &a, std::pair const &b) { // must match behavior of tileset_reader::operator<() @@ -736,23 +1000,6 @@ struct tileset_reader *begin_reading(char *fname) { return r; } -struct arg { - std::map> inputs{}; - std::map outputs{}; - - std::map *layermap = NULL; - - std::vector *header = NULL; - std::map> *mapping = NULL; - std::set *exclude = NULL; - std::set *include = NULL; - std::set *keep_layers = NULL; - std::set *remove_layers = NULL; - int ifmatched = 0; - json_object *filter = NULL; - struct tileset_reader *readers = NULL; -}; - void *join_worker(void *v) { arg *a = (arg *) v; @@ -760,7 +1007,7 @@ void *join_worker(void *v) { mvt_tile tile; for (size_t i = 0; i < ai->second.size(); i++) { - append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter); + append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), a->db, *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter, a); } ai->second.clear(); @@ -795,7 +1042,7 @@ void *join_worker(void *v) { return NULL; } -void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers) { +void dispatch_tasks(std::map> &tasks, std::vector> &layermaps, sqlite3 *outdb, const char *outdir, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::set &keep_layers, std::set &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) { pthread_t pthreads[CPUS]; std::vector args; @@ -805,6 +1052,7 @@ void dispatch_tasks(std::map> &tasks, std::vector< args[i].layermap = &layermaps[i]; args[i].header = &header; args[i].mapping = &mapping; + args[i].db = db; args[i].exclude = &exclude; args[i].include = &include; args[i].keep_layers = &keep_layers; @@ -812,6 +1060,12 @@ void dispatch_tasks(std::map> &tasks, std::vector< args[i].ifmatched = ifmatched; args[i].filter = filter; args[i].readers = readers; + args[i].minlat = *minlat; + args[i].minlon = *minlon; + args[i].maxlat = *maxlat; + args[i].maxlon = *maxlon; + args[i].minlon2 = *minlon2; + args[i].maxlon2 = *maxlon2; } size_t count = 0; @@ -844,6 +1098,13 @@ void dispatch_tasks(std::map> &tasks, std::vector< perror("pthread_join"); } + *minlat = std::min(*minlat, args[i].minlat); + *minlon = std::min(*minlon, args[i].minlon); + *maxlat = std::max(*maxlat, args[i].maxlat); + *maxlon = std::max(*maxlon, args[i].maxlon); + *minlon2 = std::min(*minlon2, args[i].minlon2); + *maxlon2 = std::max(*maxlon2, args[i].maxlon2); + for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) { if (outdb != NULL) { mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size()); @@ -945,7 +1206,7 @@ void handle_vector_layers(json_object *vector_layers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { +void decode(struct tileset_reader *readers, std::map &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector &header, std::map> &mapping, sqlite3 *db, std::set &exclude, std::set &include, int ifmatched, std::string &attribution, std::string &description, std::set &keep_layers, std::set &remove_layers, std::string &name, json_object *filter, std::map &attribute_descriptions, std::string &generator_options, std::vector *strategies) { std::vector> layermaps; for (size_t i = 0; i < CPUS; i++) { layermaps.push_back(std::map()); @@ -958,36 +1219,10 @@ void decode(struct tileset_reader *readers, std::mapall_done()) { std::pair current = readers->current(); - if (current.first.z != zoom_for_bbox) { - // Only use highest zoom for bbox calculation - // to avoid z0 always covering the world - - minlat = minlon = minlon2 = INT_MAX; - maxlat = maxlon = maxlon2 = INT_MIN; - zoom_for_bbox = current.first.z; - } - - double lat1, lon1, lat2, lon2; - tile2lonlat(current.first.x, current.first.y, current.first.z, &lon1, &lat1); - tile2lonlat(current.first.x + 1, current.first.y + 1, current.first.z, &lon2, &lat2); - minlat = min(lat2, minlat); - minlon = min(lon1, minlon); - maxlat = max(lat1, maxlat); - maxlon = max(lon2, maxlon); - - if (lon1 < 0) { - lon1 += 360; - lon2 += 360; - } - - minlon2 = min(lon1, minlon2); - maxlon2 = max(lon2, maxlon2); - if (current.first.z >= minzoom && current.first.z <= maxzoom) { zxy tile = current.first; if (tasks.count(tile) == 0) { @@ -1014,7 +1249,7 @@ void decode(struct tileset_reader *readers, std::mapzoom != current.first.z || readers->x != current.first.x || readers->y != current.first.y) { if (tasks.size() > 100 * CPUS) { - dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers); + dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2); tasks.clear(); } } @@ -1033,19 +1268,19 @@ void decode(struct tileset_reader *readers, std::mapminlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - - st->minlon2 = min(minlon2, st->minlon2); - st->maxlon2 = max(maxlon2, st->maxlon2); - st->minlat2 = min(minlat, st->minlat2); - st->maxlat2 = max(maxlat, st->maxlat2); - - dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers); + dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2); layermap = merge_layermaps(layermaps); + st->minlon = std::min(minlon, st->minlon); + st->maxlon = std::max(maxlon, st->maxlon); + st->minlat = std::min(minlat, st->minlat); + st->maxlat = std::max(maxlat, st->maxlat); + + st->minlon2 = std::min(minlon2, st->minlon2); + st->maxlon2 = std::max(maxlon2, st->maxlon2); + st->minlat2 = std::min(minlat, st->minlat2); + st->maxlat2 = std::max(maxlat, st->maxlat2); + struct tileset_reader *next; for (struct tileset_reader *r = readers; r != NULL; r = next) { next = r->next; @@ -1054,14 +1289,14 @@ void decode(struct tileset_reader *readers, std::mapdb, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { - int minz = max(sqlite3_column_int(stmt, 0), minzoom); - st->minzoom = min(st->minzoom, minz); + int minz = std::max(sqlite3_column_int(stmt, 0), minzoom); + st->minzoom = std::min(st->minzoom, minz); } sqlite3_finalize(stmt); } if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { - int maxz = min(sqlite3_column_int(stmt, 0), maxzoom); + int maxz = std::min(sqlite3_column_int(stmt, 0), maxzoom); if (!want_overzoom) { if (st->maxzoom >= 0 && maxz != st->maxzoom) { @@ -1069,7 +1304,7 @@ void decode(struct tileset_reader *readers, std::mapmaxzoom = max(st->maxzoom, maxz); + st->maxzoom = std::max(st->maxzoom, maxz); } sqlite3_finalize(stmt); } @@ -1116,20 +1351,6 @@ void decode(struct tileset_reader *readers, std::mapdb, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) { - if (sqlite3_step(stmt) == SQLITE_ROW) { - const unsigned char *s = sqlite3_column_text(stmt, 0); - if (s != NULL) { - if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) { - st->minlon = min(minlon, st->minlon); - st->maxlon = max(maxlon, st->maxlon); - st->minlat = min(minlat, st->minlat); - st->maxlat = max(maxlat, st->maxlat); - } - } - } - sqlite3_finalize(stmt); - } if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'json'", -1, &stmt, NULL) == SQLITE_OK) { if (sqlite3_step(stmt) == SQLITE_ROW) { const unsigned char *s = sqlite3_column_text(stmt, 0); @@ -1198,6 +1419,8 @@ int main(int argc, char **argv) { int filearg = 0; json_object *filter = NULL; + std::string join_sqlite_fname; + struct tileset_reader *readers = NULL; CPUS = get_num_avail_cpus(); @@ -1210,8 +1433,14 @@ int main(int argc, char **argv) { CPUS = 1; } + if (sqlite3_config(SQLITE_CONFIG_SERIALIZED) != SQLITE_OK) { + fprintf(stderr, "Could not enable sqlite3 serialized multithreading\n"); + exit(EXIT_SQLITE); + } + std::vector header; std::map> mapping; + sqlite3 *db = NULL; std::set exclude; std::set include; @@ -1235,6 +1464,7 @@ int main(int argc, char **argv) { {"exclude", required_argument, 0, 'x'}, {"exclude-all", no_argument, 0, 'X'}, {"include", required_argument, 0, 'y'}, + {"exclude-all-tile-attributes", no_argument, 0, '~'}, {"layer", required_argument, 0, 'l'}, {"exclude-layer", required_argument, 0, 'L'}, {"quiet", no_argument, 0, 'q'}, @@ -1245,6 +1475,13 @@ int main(int argc, char **argv) { {"rename-layer", required_argument, 0, 'R'}, {"read-from", required_argument, 0, 'r'}, + {"join-sqlite", required_argument, 0, '~'}, + {"join-tile-attribute", required_argument, 0, '~'}, + {"join-table-expression", required_argument, 0, '~'}, + {"join-table", required_argument, 0, '~'}, + {"join-count-limit", required_argument, 0, '~'}, + {"use-attribute-for-id", required_argument, 0, '~'}, + {"no-tile-size-limit", no_argument, &pk, 1}, {"no-tile-compression", no_argument, &pC, 1}, {"empty-csv-columns-are-null", no_argument, &pe, 1}, @@ -1429,6 +1666,24 @@ int main(int argc, char **argv) { max_tilestats_values = atoi(optarg); } else if (strcmp(opt, "unidecode-data") == 0) { unidecode_data = read_unidecode(optarg); + } else if (strcmp(opt, "join-sqlite") == 0) { + join_sqlite_fname = optarg; + if (sqlite3_open(optarg, &db) != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", optarg, sqlite3_errmsg(db)); + exit(EXIT_SQLITE); + } + } else if (strcmp(opt, "join-table") == 0) { + join_table = optarg; + } else if (strcmp(opt, "join-table-expression") == 0) { + join_table_expression = optarg; + } else if (strcmp(opt, "join-tile-attribute") == 0) { + join_tile_attribute = optarg; + } else if (strcmp(opt, "use-attribute-for-id") == 0) { + attribute_for_id = optarg; + } else if (strcmp(opt, "exclude-all-tile-attributes") == 0) { + exclude_all_tile_attributes = true; + } else if (strcmp(opt, "join-count-limit") == 0) { + join_count_limit = atoi(optarg); } else { fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt); exit(EXIT_ARGS); @@ -1510,7 +1765,7 @@ int main(int argc, char **argv) { std::string generator_options; std::vector strategies; - decode(readers, layermap, outdb, out_dir, &st, header, mapping, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); + decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies); if (set_attribution.size() != 0) { attribution = set_attribution; @@ -1541,6 +1796,10 @@ int main(int argc, char **argv) { } } + if (st.maxlon < st.minlon) { + st.maxlon = st.minlon = st.maxlat = st.minlat = st.minlon2 = st.maxlon2 = st.minlat2 = st.maxlat2 = 0; + } + if (st.maxlon - st.minlon <= st.maxlon2 - st.minlon2) { st.minlon2 = st.minlon; st.maxlon2 = st.maxlon; diff --git a/version.hpp b/version.hpp index fac06ade..71bf5f69 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "v2.73.0" +#define VERSION "v2.74.0" #endif