From 4266dc3de17c18bb854143fc13d229b6bc1b1e40 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Tue, 27 Dec 2022 11:58:40 -0800 Subject: [PATCH] Fix some ordering problems with pmtiles decode --- Makefile | 12 ++++----- decode.cpp | 17 ++++++++++++- pmtiles_file.cpp | 28 ++++++++++++++++++++ pmtiles_file.hpp | 3 ++- tests/pmtiles/hackspots.json | 38 ++++++++++++++-------------- tests/pmtiles/hackspots.pmtiles | Bin 7681 -> 0 bytes tests/pmtiles/joined_reordered.json | 38 ++++++++++++++-------------- 7 files changed, 90 insertions(+), 46 deletions(-) delete mode 100644 tests/pmtiles/hackspots.pmtiles diff --git a/Makefile b/Makefile index 3134ab2e..1f05583d 100644 --- a/Makefile +++ b/Makefile @@ -196,12 +196,12 @@ decode-test: decode-pmtiles-test: mkdir -p tests/muni/decode ./tippecanoe -q -z11 -Z11 -f -o tests/muni/decode/multi.pmtiles tests/muni/*.json - ./tippecanoe-decode -x generator -l subway tests/muni/decode/multi.pmtiles > tests/muni/decode/multi.pmtiles.json.check - ./tippecanoe-decode -x generator -l subway --integer tests/muni/decode/multi.pmtiles > tests/muni/decode/multi.pmtiles.integer.json.check - ./tippecanoe-decode -x generator -l subway --fraction tests/muni/decode/multi.pmtiles > tests/muni/decode/multi.pmtiles.fraction.json.check - ./tippecanoe-decode -x generator -c tests/muni/decode/multi.pmtiles > tests/muni/decode/multi.pmtiles.pipeline.json.check - ./tippecanoe-decode -x generator tests/muni/decode/multi.pmtiles 11 327 791 > tests/muni/decode/multi.pmtiles.onetile.json.check - ./tippecanoe-decode -x generator --stats tests/muni/decode/multi.pmtiles > tests/muni/decode/multi.pmtiles.stats.json.check + ./tippecanoe-decode -x generator -l subway tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.json.check + ./tippecanoe-decode -x generator -l subway --integer tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.integer.json.check + ./tippecanoe-decode -x generator -l subway --fraction tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.fraction.json.check + ./tippecanoe-decode -x generator -c tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.pipeline.json.check + ./tippecanoe-decode -x generator tests/muni/decode/multi.pmtiles 11 327 791 | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.onetile.json.check + ./tippecanoe-decode -x generator --stats tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.stats.json.check cmp tests/muni/decode/multi.pmtiles.json.check tests/muni/decode/multi.mbtiles.json cmp tests/muni/decode/multi.pmtiles.integer.json.check tests/muni/decode/multi.mbtiles.integer.json cmp tests/muni/decode/multi.pmtiles.fraction.json.check tests/muni/decode/multi.mbtiles.fraction.json diff --git a/decode.cpp b/decode.cpp index 6dec5c8d..93bb9404 100644 --- a/decode.cpp +++ b/decode.cpp @@ -293,7 +293,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co exit(EXIT_CLOSE); } db = pmtilesmeta2tmp(fname, pmtiles_map); - entries = pmtiles_entries_tms(pmtiles_map, minzoom, maxzoom); + entries = pmtiles_entries_zxy(pmtiles_map, minzoom, maxzoom); } else { if (sqlite3_open(fname, &db) != SQLITE_OK) { fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db)); @@ -400,7 +400,22 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set co handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode); } } else if (entries.size() > 0) { + within = 0; + for (auto const &entry : entries) { + if (!pipeline && !stats) { + if (within) { + state.json_comma_newline(); + } + within = 1; + } + if (stats) { + if (within) { + state.json_comma_newline(); + } + within = 1; + } + std::string s{pmtiles_map + entry.offset, entry.length}; handle(s, entry.z, entry.x, entry.y, to_decode, pipeline, stats, state, coordinate_mode); } diff --git a/pmtiles_file.cpp b/pmtiles_file.cpp index 1775a35c..ea001844 100644 --- a/pmtiles_file.cpp +++ b/pmtiles_file.cpp @@ -67,6 +67,34 @@ std::vector pmtiles_entries_tms(const char *pmtiles_map, int return filtered; } +struct zxycmp { + bool operator()(const pmtiles::entry_zxy &a, const pmtiles::entry_zxy &b) { + if (a.z < b.z) { + return true; + } else if (a.z == b.z) { + if (a.x < b.x) { + return true; + } else if (a.x == b.x) { + // reversed because the mbtiles SELECT + // orders by tile_row, which is TMS + if (a.y > b.y) { + return true; + } + } + } + + return false; + } +} zxycmp; + +std::vector pmtiles_entries_zxy(const char *pmtiles_map, int minzoom, int maxzoom) { + std::vector filtered; + auto all_entries = pmtiles::entries_tms(&decompress_fn, pmtiles_map); + std::copy_if(all_entries.begin(), all_entries.end(), std::back_inserter(filtered), [minzoom, maxzoom](pmtiles::entry_zxy e) { return e.z >= minzoom && e.z <= maxzoom; }); + std::sort(filtered.begin(), filtered.end(), zxycmp); + return filtered; +} + std::pair pmtiles_get_tile(const char *pmtiles_map, int z, int x, int y) { return pmtiles::get_tile(&decompress_fn, pmtiles_map, z, x, y); } diff --git a/pmtiles_file.hpp b/pmtiles_file.hpp index e8c0e4e3..359a3b96 100644 --- a/pmtiles_file.hpp +++ b/pmtiles_file.hpp @@ -10,7 +10,8 @@ void check_pmtiles(const char *filename, char **argv); void mbtiles_map_image_to_pmtiles(char *dbname, metadata m, bool tile_compression, bool quiet, bool quiet_progress); std::vector pmtiles_entries_tms(const char *pmtiles_map, int minzoom, int maxzoom); +std::vector pmtiles_entries_zxy(const char *pmtiles_map, int minzoom, int maxzoom); std::pair pmtiles_get_tile(const char *pmtiles_map, int z, int x, int y); sqlite3 *pmtilesmeta2tmp(const char *fname, const char *pmtiles_map); -#endif \ No newline at end of file +#endif diff --git a/tests/pmtiles/hackspots.json b/tests/pmtiles/hackspots.json index cfac4dbe..bf07d5d2 100644 --- a/tests/pmtiles/hackspots.json +++ b/tests/pmtiles/hackspots.json @@ -155,16 +155,6 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1464, "compressed": false }, "features": [ -{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682438, 45.569983 ] } } -, -{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675185, 45.556732 ] } } -, -{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } -] } -] } -, { "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1465, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } @@ -173,17 +163,13 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2928, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1464, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682438, 45.569983 ] } } , -{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675174, 45.556732 ] } } -] } -] } +{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675185, 45.556732 ] } } , -{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2929, "compressed": false }, "features": [ -{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521263 ] } } +{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } ] } ] } , @@ -195,9 +181,17 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2608, "y": 5856, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2929, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521263 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2928, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +, +{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675174, 45.556732 ] } } ] } ] } , @@ -207,6 +201,12 @@ ] } ] } , +{ "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2608, "y": 5856, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +] } +] } +, { "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2609, "y": 5860, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521259 ] } } diff --git a/tests/pmtiles/hackspots.pmtiles b/tests/pmtiles/hackspots.pmtiles deleted file mode 100644 index 13756da8bc8034c55a40c0e29418a66ea0408fab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7681 zcmeI%eM}Q~7zgm)wG>;h@)qn1;L*Z>@v;_#@}?6|d_zQFa~d@(^aN(1+83p2VOQFpZs!umvaN|J{~Y)2)64I7 zpC<=-E^1M1k_eR&(+IkskMzfO{gDjoe(BU7^)UgQJF#?$js64t5rQFzK&y7{Ql;!e zSVbT9s@_RlGy+T_X&gW0PDaq?`qa-K%!oJZNzjbun$f>zFt`xGb?tlK-VD zBhsg|tn6;|rw*+rLq}|j!yfjRA1#edG-a(4B`<0o;hZgWaBqz5Fj-f&D`RbP=e4V} zyzb<3(#^VtoAa+8lRf>;uWBxIY{`K;`4LOsd&o39N)hn;5?Z&6L=5HCMEcEpzSU>l zuf(fX(V1hU^fqC^^(U1+oA<7Iw|(C2Z!Wjb@eGZ5zv^=TrY+luT({8Cxic1a4Ozh- z%zu$nIkc8;F~(Tp_gJ~3ayB0Ef2itE4>v zdspSch`TIA=dihSN`BA+xp|xw|i-FGU+;H`hpq8{osXW3<(?9a0j}gBE zJ-K^#V`FXRxxiG9lDJ!SnfJ5J7T>0KoULwr(Re>EBx_3eep}mnu|?ZipLm?MsodnS z_%qE|WN%B*^01#2?qkzVX#7?jwjI)_C{aB&fo2tF7Gx+#6C6_S_R&xOe0-FZcK^h{ z0uq71tR|+yR7#>CevNdkSSEE%N0_8YK{CoT4G{_@s8q@{i4@CFDI2B|*psPdA(A+; z1c?v}6HqpdY91nrOG*(SQQAuF`D}3lvRaaarU*p}HiJ4fLLR?HfkX)6BmyD!y-*3d z0f~~wu~}3`Y@!53k?`?iDH0}Ly&6Rm#d0aicBXQ?xjZCh4#M-6B_bi~Q4b_aERl%> zDMAlqaWoRZ_vTG!yHK5{`}y#ZNJPI1tPpvo4;^b0=2O^fsM5s)P zh*J=0k`P5UV4qR3%y=n#B?i5I%GvAKoA8cuuUk%)InQ)1Vg6qq(4U{5cW_RkA^@M38KP8ePl#4 z44TrW%M*w8hX|l4ZF)f!K+{G>G{c~Exz@69XkD&#AOdLJt_>gpXxhk#W*D?O%S=8F zt{#ymYhUj@+A=7j-9G{c}(&{bR< zS`GaXhydC!J<}V9Hb_6@ZA3E++6C?-1cx@v?Qqmb0IiRg<%C1)<~2JR(F}uD z(E1!}h&Z(Cob8ivXc|r_#fWAYv?K6KAuthXBMy^sXh%#7EO2P~rfMrAnqkmxkVb$I zK&vwSWik%!2CW=CM8Q8IrL-X@=0j)#v&a8=xHzHNp2hfVj|h>lsU3-;w4bHF0Wf3& zL)g$xZP!a-*25Fn?HeYoZ^QUhe~%z+{HdLFDk=S-Z?w0Kj74pH#g)j#vvW7P_uc7DmFZXAn z+n?1<&}A@0gh^QARoR#G-_|ZW=bH)R9#37o0vBf4-xbEc_4yAt_3OjlmcTH^aXqls Xi>dhMPtS?G)Opz)y5>LU{4M`?B_3~~ diff --git a/tests/pmtiles/joined_reordered.json b/tests/pmtiles/joined_reordered.json index 53fcd1bb..7d9fa13f 100644 --- a/tests/pmtiles/joined_reordered.json +++ b/tests/pmtiles/joined_reordered.json @@ -155,16 +155,6 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1464, "compressed": false }, "features": [ -{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682438, 45.569983 ] } } -, -{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675185, 45.556732 ] } } -, -{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } -] } -] } -, { "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1465, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } @@ -173,17 +163,13 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2928, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 12, "x": 652, "y": 1464, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682438, 45.569983 ] } } , -{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675174, 45.556732 ] } } -] } -] } +{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675185, 45.556732 ] } } , -{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2929, "compressed": false }, "features": [ -{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521263 ] } } +{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654972, 45.521263 ] } } ] } ] } , @@ -195,9 +181,17 @@ ] } ] } , -{ "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2608, "y": 5856, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2929, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521263 ] } } +] } +] } +, +{ "type": "FeatureCollection", "properties": { "zoom": 13, "x": 1304, "y": 2928, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +, +{ "type": "Feature", "properties": { "Name": "Albina Press", "Address": "4637 N Albina Ave Portland, OR 97217", "Notes": "usually busy, outlets on side wall only" }, "geometry": { "type": "Point", "coordinates": [ -122.675174, 45.556732 ] } } ] } ] } , @@ -207,6 +201,12 @@ ] } ] } , +{ "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2608, "y": 5856, "compressed": false }, "features": [ +{ "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ +{ "type": "Feature", "properties": { "Name": "Arbor Lodge", "Address": "1507 N Rosa Parks Way Portland, OR 97217", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.682427, 45.569975 ] } } +] } +] } +, { "type": "FeatureCollection", "properties": { "zoom": 14, "x": 2609, "y": 5860, "compressed": false }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "hackspots", "version": 2, "extent": 4096 }, "features": [ { "type": "Feature", "properties": { "Name": "Three Friends Coffeehouse", "Address": "201 SE 12th Ave, Portland, OR 97214", "Notes": "" }, "geometry": { "type": "Point", "coordinates": [ -122.654961, 45.521259 ] } }