mirror of
https://github.com/felt/tippecanoe.git
synced 2026-11-22 16:31:18 +01:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49762ee812 | ||
|
|
0d944ede52 | ||
|
|
434817bde9 | ||
|
|
da7bbbd6ea | ||
|
|
e43d6dacdd | ||
|
|
afc7ca01a2 | ||
|
|
b3c3078480 | ||
|
|
329dd86bfd | ||
|
|
3d714e32b0 | ||
|
|
e1bbc843be | ||
|
|
fd6c9ec94c | ||
|
|
7371072e45 | ||
|
|
0fa758c1e3 | ||
|
|
f2127fec97 | ||
|
|
71df08511e | ||
|
|
efa209d175 | ||
|
|
9f97397e21 | ||
|
|
61b5475015 |
@@ -1,3 +1,21 @@
|
||||
* Take feature ordering into account when dropping points in the low zooms
|
||||
|
||||
# 2.24.0
|
||||
|
||||
>>>>>>> main
|
||||
* Add --cluster-maxzoom option to limit zoom levels that receive clustering
|
||||
* Add `point_count_abbreviated` attribute to clustered features, for consistency with supercluster
|
||||
* Makefile changes to support FreeBSD
|
||||
* Add -r option to tile-join to provide a file containing a list of input files
|
||||
* Add antimeridian_adjusted_bounds field to tileset metadata
|
||||
|
||||
## 2.23.0
|
||||
|
||||
* Remove the concept of "separate metadata." Features now always directly reference their keys and values rather than going through a second level of indirection.
|
||||
* Limit the size of the string pool to 1/5 the size of memory, to prevent thrashing during feature ingestion.
|
||||
* Avoid using writeable memory maps. Instead, explicitly copy data in and out of memory.
|
||||
* Compress streams of features in the temporary files, to reduce disk usage and I/O latency
|
||||
|
||||
## 2.22.0
|
||||
|
||||
* Speed up feature dropping by removing unnecessary search for other small features
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
PREFIX ?= /usr/local
|
||||
MANDIR ?= $(PREFIX)/share/man/man1/
|
||||
BUILDTYPE ?= Release
|
||||
SHELL = /bin/bash
|
||||
SHELL = /bin/sh
|
||||
|
||||
|
||||
# inherit from env if set
|
||||
CC := $(CC)
|
||||
CXX := $(CXX)
|
||||
CFLAGS := $(CFLAGS)
|
||||
CXXFLAGS := $(CXXFLAGS) -std=c++11
|
||||
CXXFLAGS := $(CXXFLAGS) -std=c++17
|
||||
LDFLAGS := $(LDFLAGS)
|
||||
WARNING_FLAGS := -Wall -Wshadow -Wsign-compare -Wextra -Wunreachable-code -Wuninitialized -Wshadow
|
||||
RELEASE_FLAGS := -O3 -DNDEBUG
|
||||
DEBUG_FLAGS := -O0 -DDEBUG -fno-inline-functions -fno-omit-frame-pointer
|
||||
|
||||
|
||||
OS := $(shell uname -s)
|
||||
ifeq ($(OS),FreeBSD)
|
||||
ADVSHELL = /usr/local/bin/bash
|
||||
else
|
||||
ADVSHELL = /bin/bash
|
||||
endif
|
||||
|
||||
|
||||
ifeq ($(BUILDTYPE),Release)
|
||||
FINAL_FLAGS := -g $(WARNING_FLAGS) $(RELEASE_FLAGS)
|
||||
else
|
||||
@@ -47,7 +57,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
|
||||
INCLUDES = -I/usr/local/include -I.
|
||||
LIBS = -L/usr/local/lib
|
||||
|
||||
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o
|
||||
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
|
||||
|
||||
tippecanoe-enumerate: enumerate.o
|
||||
@@ -56,7 +66,7 @@ tippecanoe-enumerate: enumerate.o
|
||||
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
|
||||
|
||||
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o
|
||||
tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o
|
||||
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
|
||||
|
||||
tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
|
||||
@@ -111,7 +121,7 @@ fewer-tests: tippecanoe tippecanoe-decode geobuf-test raw-tiles-test parallel-te
|
||||
cmp $@.out $(patsubst %.checkbuf,%,$@)
|
||||
rm $@.out $@.mbtiles
|
||||
|
||||
parallel-test:
|
||||
parallel-test: $(eval SHELL:=$(ADVSHELL))
|
||||
mkdir -p tests/parallel
|
||||
perl -e 'for ($$i = 0; $$i < 20; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; $$k = rand(1); $$v = rand(1); print "{ \"type\": \"Feature\", \"properties\": { \"yes\": \"no\", \"who\": 1, \"$$k\": \"$$v\" }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in1.json
|
||||
perl -e 'for ($$i = 0; $$i < 300000; $$i++) { $$lon = rand(360) - 180; $$lat = rand(180) - 90; print "{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Point\", \"coordinates\": [ $$lon, $$lat ] } }\n"; }' > tests/parallel/in2.json
|
||||
@@ -329,6 +339,15 @@ join-test: tile-join
|
||||
./tippecanoe-decode -x generator tests/join-population/concat.mbtiles > tests/join-population/concat.mbtiles.json.check
|
||||
cmp tests/join-population/concat.mbtiles.json.check tests/join-population/concat.mbtiles.json
|
||||
rm tests/join-population/concat.mbtiles.json.check tests/join-population/concat.mbtiles tests/join-population/macarthur.mbtiles
|
||||
# Test reading list of input files from file
|
||||
./tippecanoe -q -f -Z5 -z10 -o tests/readfile/macarthur.mbtiles -l macarthur1 tests/join-population/macarthur.json
|
||||
./tippecanoe -q -f -Z5 -z10 -o tests/readfile/macarthur2.mbtiles -l macarthur2 tests/join-population/macarthur2.json
|
||||
./tile-join -q -R macarthur1:one --rename-layer=macarthur2:two -f -o tests/readfile/renamed.mbtiles tests/readfile/macarthur.mbtiles tests/readfile/macarthur2.mbtiles
|
||||
./tippecanoe-decode -x generator -x generator_options tests/readfile/renamed.mbtiles > tests/readfile/renamed.mbtiles.json.check
|
||||
./tile-join -q -R macarthur1:one --rename-layer=macarthur2:two -f -r tests/readfile/readfile.list -o tests/readfile/readfile.mbtiles
|
||||
./tippecanoe-decode -x generator -x generator_options tests/readfile/readfile.mbtiles > tests/readfile/readfile.mbtiles.json.check
|
||||
cmp tests/readfile/renamed.mbtiles.json.check tests/readfile/readfile.mbtiles.json.check
|
||||
rm tests/readfile/renamed.mbtiles.json.check tests/readfile/readfile.mbtiles.json.check tests/readfile/readfile.mbtiles tests/readfile/renamed.mbtiles
|
||||
#`
|
||||
# Make sure empty tilesets work
|
||||
#
|
||||
|
||||
@@ -456,6 +456,8 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
|
||||
* `-al` or `--drop-lines`: Let "dot" dropping at lower zooms apply to lines too
|
||||
* `-ap` or `--drop-polygons`: Let "dot" dropping at lower zooms apply to polygons too
|
||||
* `-K` _distance_ or `--cluster-distance=`_distance_: Cluster points (as with `--cluster-densest-as-needed`, but without the experimental discovery process) that are approximately within _distance_ of each other. The units are tile coordinates within a nominally 256-pixel tile, so the maximum value of 255 allows only one feature per tile. Values around 10 are probably appropriate for typical marker sizes. See `--cluster-densest-as-needed` below for behavior.
|
||||
* `-k` _zoom_ or `--cluster-maxzoom=`_zoom_: Max zoom on which to cluster points if clustering is enabled.
|
||||
* `-kg` or `--cluster-maxzoom=g`: Set `--cluster-maxzoom=` to `maxzoom - 1` so that all features are visible at the maximum zoom level.
|
||||
|
||||
### Dropping a fraction of features to keep under tile size limits
|
||||
|
||||
@@ -757,6 +759,7 @@ The options are:
|
||||
* `-o` *out.mbtiles*, *out.pmtiles* or `--output=`*out.mbtiles*: Write the new tiles to the specified .mbtiles file.
|
||||
* `-e` *directory* or `--output-to-directory=`*directory*: Write the new tiles to the specified directory instead of to an mbtiles file.
|
||||
* `-f` or `--force`: Remove *out.mbtiles* if it already exists.
|
||||
* `-r` or `--read-from`: list of input mbtiles to read from.
|
||||
|
||||
### Tileset description and attribution
|
||||
|
||||
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
#ifdef __APPLE__
|
||||
#define _DARWIN_UNLIMITED_STREAMS
|
||||
#endif
|
||||
|
||||
#include "compression.hpp"
|
||||
#include "errors.hpp"
|
||||
#include "protozero/varint.hpp"
|
||||
#include "serial.hpp"
|
||||
|
||||
void decompressor::begin() {
|
||||
within = true;
|
||||
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = inflateInit(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int decompressor::fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos) {
|
||||
zs.next_out = (Bytef *) p;
|
||||
zs.avail_out = size * nmemb;
|
||||
|
||||
while (zs.avail_out > 0) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
if (n == 0) {
|
||||
if (within) {
|
||||
fprintf(stderr, "Reached EOF while decompressing\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
size_t avail_before = zs.avail_in;
|
||||
|
||||
if (within) {
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
*geompos += avail_before - zs.avail_in;
|
||||
|
||||
if (d == Z_OK) {
|
||||
// it made some progress
|
||||
} else if (d == Z_STREAM_END) {
|
||||
// it may have made some progress and now we are done
|
||||
within = false;
|
||||
break;
|
||||
} else {
|
||||
fprintf(stderr, "decompression error %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
} else {
|
||||
size_t n = std::min(zs.avail_in, zs.avail_out);
|
||||
memcpy(zs.next_out, zs.next_in, n);
|
||||
*geompos += n;
|
||||
|
||||
zs.avail_out -= n;
|
||||
zs.avail_in -= n;
|
||||
zs.next_out += n;
|
||||
zs.next_in += n;
|
||||
}
|
||||
}
|
||||
|
||||
return (size * nmemb - zs.avail_out) / size;
|
||||
}
|
||||
|
||||
void decompressor::end(std::atomic<long long> *geompos) {
|
||||
// "within" means that we haven't received end-of-stream yet,
|
||||
// so consume more compressed data until we get there.
|
||||
// This can be necessary if the caller knows that it is at
|
||||
// the end of the feature stream (because it got a 0-length
|
||||
// feature) but the decompressor doesn't know yet.
|
||||
|
||||
if (within) {
|
||||
while (true) {
|
||||
if (zs.avail_in == 0) {
|
||||
size_t n = ::fread((Bytef *) buf.c_str(), sizeof(char), buf.size(), fp);
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = n;
|
||||
}
|
||||
|
||||
zs.avail_out = 0;
|
||||
|
||||
size_t avail_before = zs.avail_in;
|
||||
int d = inflate(&zs, Z_NO_FLUSH);
|
||||
*geompos += avail_before - zs.avail_in;
|
||||
|
||||
if (d == Z_STREAM_END) {
|
||||
break;
|
||||
}
|
||||
if (d == Z_OK) {
|
||||
continue;
|
||||
}
|
||||
|
||||
fprintf(stderr, "decompression: got %d, not Z_STREAM_END\n", d);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
within = false;
|
||||
}
|
||||
|
||||
int d = inflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "end decompression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
int decompressor::deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (1) {
|
||||
char c;
|
||||
if (fread(&c, sizeof(char), 1, geompos) != 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_long_long(long long *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long zigzag = 0;
|
||||
int ret = deserialize_ulong_long(&zigzag, geompos);
|
||||
*n = protozero::decode_zigzag64(zigzag);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_int(int *n, std::atomic<long long> *geompos) {
|
||||
long long ll = 0;
|
||||
int ret = deserialize_long_long(&ll, geompos);
|
||||
*n = ll;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int decompressor::deserialize_uint(unsigned *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long v;
|
||||
deserialize_ulong_long(&v, geompos);
|
||||
*n = v;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void compressor::begin() {
|
||||
zs.zalloc = NULL;
|
||||
zs.zfree = NULL;
|
||||
zs.opaque = NULL;
|
||||
zs.msg = (char *) "";
|
||||
|
||||
int d = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "initialize compression: %d %s\n", d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::compressor::end(std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(5000);
|
||||
|
||||
if (zs.avail_in != 0) {
|
||||
fprintf(stderr, "compression end called with data available\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
|
||||
while (true) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_FINISH);
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
|
||||
if (d == Z_OK || d == Z_BUF_ERROR) {
|
||||
// it can take several calls to flush out all the buffered data
|
||||
continue;
|
||||
}
|
||||
|
||||
if (d != Z_STREAM_END) {
|
||||
fprintf(stderr, "%s: finish compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflateEnd(&zs);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: end compression: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
}
|
||||
|
||||
int compressor::fclose() {
|
||||
return ::fclose(fp);
|
||||
}
|
||||
|
||||
void compressor::fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname) {
|
||||
std::string buf;
|
||||
buf.resize(size * nmemb * 2 + 200);
|
||||
|
||||
zs.next_in = (Bytef *) p;
|
||||
zs.avail_in = size * nmemb;
|
||||
|
||||
while (zs.avail_in > 0) {
|
||||
zs.next_out = (Bytef *) buf.c_str();
|
||||
zs.avail_out = buf.size();
|
||||
|
||||
int d = deflate(&zs, Z_NO_FLUSH);
|
||||
if (d != Z_OK) {
|
||||
fprintf(stderr, "%s: deflate: %d %s\n", fname, d, zs.msg);
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
::fwrite_check(buf.c_str(), sizeof(char), zs.next_out - (Bytef *) buf.c_str(), fp, fpos, fname);
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
while (1) {
|
||||
unsigned char b = val & 0x7F;
|
||||
if ((val >> 7) != 0) {
|
||||
b |= 0x80;
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
val >>= 7;
|
||||
} else {
|
||||
fwrite_check((const char *) &b, 1, 1, fpos, fname);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void compressor::serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname) {
|
||||
unsigned long long zigzag = protozero::encode_zigzag64(val);
|
||||
|
||||
serialize_ulong_long(zigzag, fpos, fname);
|
||||
}
|
||||
|
||||
void compressor::serialize_int(int val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_long_long(val, fpos, fname);
|
||||
}
|
||||
|
||||
void compressor::serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname) {
|
||||
serialize_ulong_long(val, fpos, fname);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
#ifdef __APPLE__
|
||||
#define _DARWIN_UNLIMITED_STREAMS
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <atomic>
|
||||
#include <zlib.h>
|
||||
|
||||
struct decompressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
std::string buf;
|
||||
|
||||
// from begin() to receiving end-of-stream
|
||||
bool within = false;
|
||||
|
||||
decompressor(FILE *f) {
|
||||
fp = f;
|
||||
buf.resize(5000);
|
||||
|
||||
zs.next_in = (Bytef *) buf.c_str();
|
||||
zs.avail_in = 0;
|
||||
}
|
||||
|
||||
decompressor() {
|
||||
}
|
||||
|
||||
void begin();
|
||||
int fread(void *p, size_t size, size_t nmemb, std::atomic<long long> *geompos);
|
||||
void end(std::atomic<long long> *geompos);
|
||||
int deserialize_ulong_long(unsigned long long *zigzag, std::atomic<long long> *geompos);
|
||||
int deserialize_long_long(long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_int(int *n, std::atomic<long long> *geompos);
|
||||
int deserialize_uint(unsigned *n, std::atomic<long long> *geompos);
|
||||
};
|
||||
|
||||
struct compressor {
|
||||
FILE *fp = NULL;
|
||||
z_stream zs;
|
||||
|
||||
compressor(FILE *f) {
|
||||
fp = f;
|
||||
}
|
||||
|
||||
compressor() {
|
||||
}
|
||||
|
||||
void begin();
|
||||
void end(std::atomic<long long> *fpos, const char *fname);
|
||||
int fclose();
|
||||
void fwrite_check(const char *p, size_t size, size_t nmemb, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_ulong_long(unsigned long long val, std::atomic<long long> *fpos, const char *fname);
|
||||
|
||||
void serialize_long_long(long long val, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_int(int val, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_uint(unsigned val, std::atomic<long long> *fpos, const char *fname);
|
||||
};
|
||||
@@ -310,6 +310,7 @@ void dir_write_metadata(const char *outdir, const metadata &m) {
|
||||
out(state, "maxzoom", std::to_string(m.maxzoom));
|
||||
out(state, "center", std::to_string(m.center_lon) + "," + std::to_string(m.center_lat) + "," + std::to_string(m.center_z));
|
||||
out(state, "bounds", std::to_string(m.minlon) + "," + std::to_string(m.minlat) + "," + std::to_string(m.maxlon) + "," + std::to_string(m.maxlat));
|
||||
out(state, "antimeridian_adjusted_bounds", std::to_string(m.minlon2) + "," + std::to_string(m.minlat2) + "," + std::to_string(m.maxlon2) + "," + std::to_string(m.maxlat2));
|
||||
out(state, "type", m.type);
|
||||
if (m.attribution.size() > 0) {
|
||||
out(state, "attribution", m.attribution);
|
||||
|
||||
+13
-11
@@ -25,7 +25,7 @@
|
||||
|
||||
static int clip(double *x0, double *y0, double *x1, double *y1, double xmin, double ymin, double xmax, double ymax);
|
||||
|
||||
drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y) {
|
||||
drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y) {
|
||||
drawvec out;
|
||||
|
||||
bbox[0] = LLONG_MAX;
|
||||
@@ -38,10 +38,7 @@ drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsi
|
||||
while (1) {
|
||||
draw d;
|
||||
|
||||
if (!deserialize_byte_io(meta, &d.op, geompos)) {
|
||||
fprintf(stderr, "Internal error: Unexpected end of file in geometry\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
deserialize_byte(meta, &d.op);
|
||||
if (d.op == VT_END) {
|
||||
break;
|
||||
}
|
||||
@@ -49,8 +46,8 @@ drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsi
|
||||
if (d.op == VT_MOVETO || d.op == VT_LINETO) {
|
||||
long long dx, dy;
|
||||
|
||||
deserialize_long_long_io(meta, &dx, geompos);
|
||||
deserialize_long_long_io(meta, &dy, geompos);
|
||||
deserialize_long_long(meta, &dx);
|
||||
deserialize_long_long(meta, &dy);
|
||||
|
||||
wx += dx * (1 << geometry_scale);
|
||||
wy += dy * (1 << geometry_scale);
|
||||
@@ -1541,7 +1538,8 @@ struct sorty {
|
||||
|
||||
struct sorty_sorter {
|
||||
int kind;
|
||||
sorty_sorter(int k) : kind(k) {};
|
||||
sorty_sorter(int k)
|
||||
: kind(k){};
|
||||
|
||||
bool operator()(const sorty &a, const sorty &b) const {
|
||||
long long xa, ya, xb, yb;
|
||||
@@ -1552,13 +1550,13 @@ struct sorty_sorter {
|
||||
|
||||
xb = b.x;
|
||||
yb = b.y;
|
||||
} else if (kind == 1) { // X first
|
||||
} else if (kind == 1) { // X first
|
||||
xa = a.y;
|
||||
ya = a.x;
|
||||
|
||||
xb = b.y;
|
||||
yb = b.x;
|
||||
} else if (kind == 2) { // diagonal
|
||||
} else if (kind == 2) { // diagonal
|
||||
xa = a.x + a.y;
|
||||
ya = a.x - a.y;
|
||||
|
||||
@@ -1775,7 +1773,11 @@ drawvec polygon_to_anchor(const drawvec &geom) {
|
||||
if (goodness <= 0) {
|
||||
double lon, lat;
|
||||
tile2lonlat(d.x, d.y, 32, &lon, &lat);
|
||||
fprintf(stderr, "could not find label point: %s %f,%f\n", kind, lat, lon);
|
||||
|
||||
static std::atomic<long long> warned(0);
|
||||
if (warned++ < 10) {
|
||||
fprintf(stderr, "could not find good label point: %s %f,%f\n", kind, lat, lon);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ struct draw {
|
||||
typedef std::vector<draw> drawvec;
|
||||
struct serial_feature;
|
||||
|
||||
drawvec decode_geometry(FILE *meta, std::atomic<long long> *geompos, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y);
|
||||
drawvec decode_geometry(char **meta, int z, unsigned tx, unsigned ty, long long *bbox, unsigned initial_x, unsigned initial_y);
|
||||
void to_tile_scale(drawvec &geom, int z, int detail);
|
||||
drawvec from_tile_scale(drawvec const &geom, int z, int detail);
|
||||
drawvec remove_noop(drawvec geom, int type, int shift);
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
#include <map>
|
||||
#include <cmath>
|
||||
|
||||
#ifdef __APPLE__
|
||||
#if defined(__APPLE__) || defined(__FreeBSD__)
|
||||
#include <sys/types.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/param.h>
|
||||
@@ -82,6 +82,7 @@ size_t max_tile_size = 500000;
|
||||
size_t max_tile_features = 200000;
|
||||
int cluster_distance = 0;
|
||||
int tiny_polygon_size = 2;
|
||||
int cluster_maxzoom = MAX_ZOOM;
|
||||
long justx = -1, justy = -1;
|
||||
std::string attribute_for_id = "";
|
||||
size_t limit_tile_feature_count = 0;
|
||||
@@ -105,6 +106,7 @@ struct source {
|
||||
size_t CPUS;
|
||||
size_t TEMP_FILES;
|
||||
long long MAX_FILES;
|
||||
size_t memsize;
|
||||
static long long diskfree;
|
||||
char **av;
|
||||
|
||||
@@ -113,9 +115,9 @@ std::vector<clipbbox> clipbboxes;
|
||||
void checkdisk(std::vector<struct reader> *r) {
|
||||
long long used = 0;
|
||||
for (size_t i = 0; i < r->size(); i++) {
|
||||
// Meta, pool, and tree are used once.
|
||||
// Pool and tree are used once.
|
||||
// Geometry and index will be duplicated during sorting and tiling.
|
||||
used += (*r)[i].metapos + 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->len + (*r)[i].treefile->len;
|
||||
used += 2 * (*r)[i].geompos + 2 * (*r)[i].indexpos + (*r)[i].poolfile->off + (*r)[i].treefile->off;
|
||||
}
|
||||
|
||||
static int warned = 0;
|
||||
@@ -326,8 +328,11 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
|
||||
while (head != NULL) {
|
||||
struct index ix = *((struct index *) (map + head->start));
|
||||
long long pos = *geompos;
|
||||
fwrite_check(geom_map + ix.start, 1, ix.end - ix.start, geom_out, "merge geometry");
|
||||
*geompos += ix.end - ix.start;
|
||||
|
||||
// MAGIC: This knows that the feature minzoom is the last byte of the serialized feature
|
||||
// and is writing one byte less and then adding the byte for the minzoom.
|
||||
|
||||
fwrite_check(geom_map + ix.start, 1, ix.end - ix.start - 1, geom_out, geompos, "merge geometry");
|
||||
int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, gamma);
|
||||
serialize_byte(geom_out, feature_minzoom, geompos, "merge geometry");
|
||||
|
||||
@@ -335,12 +340,14 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map,
|
||||
*progress += (ix.end - ix.start) * 3 / 4;
|
||||
if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
|
||||
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
|
||||
fflush(stderr);
|
||||
*progress_reported = 100 * *progress / *progress_max;
|
||||
}
|
||||
|
||||
ix.start = pos;
|
||||
ix.end = *geompos;
|
||||
fwrite_check(&ix, bytes, 1, indexfile, "merge temporary");
|
||||
std::atomic<long long> indexpos;
|
||||
fwrite_check(&ix, bytes, 1, indexfile, &indexpos, "merge temporary");
|
||||
head->start += bytes;
|
||||
|
||||
struct mergelist *m = head;
|
||||
@@ -382,32 +389,25 @@ void *run_sort(void *v) {
|
||||
a->merges[start / a->unit].end = end;
|
||||
a->merges[start / a->unit].next = NULL;
|
||||
|
||||
// MAP_PRIVATE to avoid disk writes if it fits in memory
|
||||
void *map = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_PRIVATE, a->indexfd, start);
|
||||
if (map == MAP_FAILED) {
|
||||
perror("mmap in run_sort");
|
||||
exit(EXIT_MEMORY);
|
||||
// Read section of index into memory to sort and then use pwrite()
|
||||
// to write it back out rather than sorting in mapped memory,
|
||||
// because writable mapped memory seems to have bad performance
|
||||
// problems on ECS (and maybe in containers in general)?
|
||||
|
||||
std::string s;
|
||||
s.resize(end - start);
|
||||
|
||||
if (pread(a->indexfd, (void *) s.c_str(), end - start, start) != end - start) {
|
||||
fprintf(stderr, "pread(index): %s\n", strerror(errno));
|
||||
exit(EXIT_READ);
|
||||
}
|
||||
madvise(map, end - start, MADV_RANDOM);
|
||||
madvise(map, end - start, MADV_WILLNEED);
|
||||
|
||||
qsort(map, (end - start) / a->bytes, a->bytes, indexcmp);
|
||||
qsort((void *) s.c_str(), (end - start) / a->bytes, a->bytes, indexcmp);
|
||||
|
||||
// Sorting and then copying avoids disk access to
|
||||
// write out intermediate stages of the sort.
|
||||
|
||||
void *map2 = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_SHARED, a->indexfd, start);
|
||||
if (map2 == MAP_FAILED) {
|
||||
perror("mmap (write)");
|
||||
exit(EXIT_MEMORY);
|
||||
if (pwrite(a->indexfd, s.c_str(), end - start, start) != end - start) {
|
||||
fprintf(stderr, "pwrite(index): %s\n", strerror(errno));
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
madvise(map2, end - start, MADV_SEQUENTIAL);
|
||||
|
||||
memcpy(map2, map, end - start);
|
||||
|
||||
// No madvise, since caller will want the sorted data
|
||||
munmap(map, end - start);
|
||||
munmap(map2, end - start);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
@@ -788,20 +788,21 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
|
||||
unsigned long long which = (ix.ix << prefix) >> (64 - splitbits);
|
||||
long long pos = sub_geompos[which];
|
||||
|
||||
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], "geom");
|
||||
sub_geompos[which] += ix.end - ix.start;
|
||||
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfiles[which], &sub_geompos[which], "geom");
|
||||
|
||||
// Count this as a 25%-accomplishment, since we will copy again
|
||||
*progress += (ix.end - ix.start) / 4;
|
||||
if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
|
||||
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
|
||||
fflush(stderr);
|
||||
*progress_reported = 100 * *progress / *progress_max;
|
||||
}
|
||||
|
||||
ix.start = pos;
|
||||
ix.end = sub_geompos[which];
|
||||
|
||||
fwrite_check(&ix, sizeof(struct index), 1, indexfiles[which], "index");
|
||||
std::atomic<long long> indexpos;
|
||||
fwrite_check(&ix, sizeof(struct index), 1, indexfiles[which], &indexpos, "index");
|
||||
}
|
||||
|
||||
madvise(indexmap, indexst.st_size, MADV_DONTNEED);
|
||||
@@ -958,8 +959,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
|
||||
struct index ix = indexmap[a];
|
||||
long long pos = *geompos_out;
|
||||
|
||||
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, "geom");
|
||||
*geompos_out += ix.end - ix.start;
|
||||
fwrite_check(geommap + ix.start, ix.end - ix.start, 1, geomfile, geompos_out, "geom");
|
||||
int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, gamma);
|
||||
serialize_byte(geomfile, feature_minzoom, geompos_out, "merge geometry");
|
||||
|
||||
@@ -967,12 +967,14 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
|
||||
*progress += (ix.end - ix.start) * 3 / 4;
|
||||
if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) {
|
||||
fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max);
|
||||
fflush(stderr);
|
||||
*progress_reported = 100 * *progress / *progress_max;
|
||||
}
|
||||
|
||||
ix.start = pos;
|
||||
ix.end = *geompos_out;
|
||||
fwrite_check(&ix, sizeof(struct index), 1, indexfile, "index");
|
||||
std::atomic<long long> indexpos;
|
||||
fwrite_check(&ix, sizeof(struct index), 1, indexfile, &indexpos, "index");
|
||||
}
|
||||
|
||||
madvise(indexmap, indexst.st_size, MADV_DONTNEED);
|
||||
@@ -1028,17 +1030,8 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
|
||||
}
|
||||
}
|
||||
|
||||
void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
|
||||
// Run through the index and geometry for each reader,
|
||||
// splitting the contents out by index into as many
|
||||
// sub-files as we can write to simultaneously.
|
||||
|
||||
// Then sort each of those by index, recursively if it is
|
||||
// too big to fit in memory.
|
||||
|
||||
// Then concatenate each of the sub-outputs into a final output.
|
||||
|
||||
long long mem;
|
||||
static size_t calc_memsize() {
|
||||
size_t mem;
|
||||
|
||||
#ifdef __APPLE__
|
||||
int64_t hw_memsize;
|
||||
@@ -1059,6 +1052,21 @@ void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FI
|
||||
mem = (long long) pages * pagesize;
|
||||
#endif
|
||||
|
||||
return mem;
|
||||
}
|
||||
|
||||
void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
|
||||
// Run through the index and geometry for each reader,
|
||||
// splitting the contents out by index into as many
|
||||
// sub-files as we can write to simultaneously.
|
||||
|
||||
// Then sort each of those by index, recursively if it is
|
||||
// too big to fit in memory.
|
||||
|
||||
// Then concatenate each of the sub-outputs into a final output.
|
||||
|
||||
long long mem = memsize;
|
||||
|
||||
// Just for code coverage testing. Deeply recursive sorting is very slow
|
||||
// compared to sorting in memory.
|
||||
if (additional[A_PREFER_RADIX_SORT]) {
|
||||
@@ -1066,7 +1074,7 @@ void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FI
|
||||
}
|
||||
|
||||
long long availfiles = MAX_FILES - 2 * nreaders // each reader has a geom and an index
|
||||
- 4 // pool, meta, mbtiles, mbtiles journal
|
||||
- 3 // pool, mbtiles, mbtiles journal
|
||||
- 4 // top-level geom and index output, both FILE and fd
|
||||
- 3; // stdin, stdout, stderr
|
||||
|
||||
@@ -1105,7 +1113,7 @@ void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FI
|
||||
}
|
||||
}
|
||||
|
||||
void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) {
|
||||
void choose_first_zoom(long long *file_bbox, long long *file_bbox1, long long *file_bbox2, std::vector<struct reader> &readers, unsigned *iz, unsigned *ix, unsigned *iy, int minzoom, int buffer) {
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (readers[i].file_bbox[0] < file_bbox[0]) {
|
||||
file_bbox[0] = readers[i].file_bbox[0];
|
||||
@@ -1119,6 +1127,16 @@ void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers
|
||||
if (readers[i].file_bbox[3] > file_bbox[3]) {
|
||||
file_bbox[3] = readers[i].file_bbox[3];
|
||||
}
|
||||
|
||||
file_bbox1[0] = std::min(file_bbox1[0], readers[i].file_bbox1[0]);
|
||||
file_bbox1[1] = std::min(file_bbox1[1], readers[i].file_bbox1[1]);
|
||||
file_bbox1[2] = std::max(file_bbox1[2], readers[i].file_bbox1[2]);
|
||||
file_bbox1[3] = std::max(file_bbox1[3], readers[i].file_bbox1[3]);
|
||||
|
||||
file_bbox2[0] = std::min(file_bbox2[0], readers[i].file_bbox2[0]);
|
||||
file_bbox2[1] = std::min(file_bbox2[1], readers[i].file_bbox2[1]);
|
||||
file_bbox2[2] = std::max(file_bbox2[2], readers[i].file_bbox2[2]);
|
||||
file_bbox2[3] = std::max(file_bbox2[3], readers[i].file_bbox2[3]);
|
||||
}
|
||||
|
||||
// If the bounding box extends off the plane on either side,
|
||||
@@ -1156,7 +1174,7 @@ void choose_first_zoom(long long *file_bbox, std::vector<struct reader> &readers
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) {
|
||||
std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, json_object *filter, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, long long *file_bbox1, long long *file_bbox2, const char *prefilter, const char *postfilter, const char *description, bool guess_maxzoom, bool guess_cluster_maxzoom, std::map<std::string, int> const *attribute_types, const char *pgm, std::map<std::string, attribute_op> const *attribute_accum, std::map<std::string, std::string> const &attribute_descriptions, std::string const &commandline, int minimum_maxzoom) {
|
||||
int ret = EXIT_SUCCESS;
|
||||
|
||||
std::vector<struct reader> readers;
|
||||
@@ -1164,23 +1182,16 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
struct reader *r = &readers[i];
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
char treename[strlen(tmpdir) + strlen("/tree.XXXXXXXX") + 1];
|
||||
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1];
|
||||
char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1];
|
||||
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
sprintf(poolname, "%s%s", tmpdir, "/pool.XXXXXXXX");
|
||||
sprintf(treename, "%s%s", tmpdir, "/tree.XXXXXXXX");
|
||||
sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX");
|
||||
sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX");
|
||||
|
||||
r->metafd = mkstemp_cloexec(metaname);
|
||||
if (r->metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->poolfd = mkstemp_cloexec(poolname);
|
||||
if (r->poolfd < 0) {
|
||||
perror(poolname);
|
||||
@@ -1202,11 +1213,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
r->metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (r->metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->poolfile = memfile_open(r->poolfd);
|
||||
if (r->poolfile == NULL) {
|
||||
perror(poolname);
|
||||
@@ -1227,11 +1233,9 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror(indexname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
r->metapos = 0;
|
||||
r->geompos = 0;
|
||||
r->indexpos = 0;
|
||||
|
||||
unlink(metaname);
|
||||
unlink(poolname);
|
||||
unlink(treename);
|
||||
unlink(geomname);
|
||||
@@ -1242,8 +1246,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
struct stringpool p;
|
||||
memfile_write(r->treefile, &p, sizeof(struct stringpool));
|
||||
}
|
||||
// Keep metadata file from being completely empty if no attributes
|
||||
serialize_int(r->metafile, 0, &r->metapos, "meta");
|
||||
|
||||
r->file_bbox[0] = r->file_bbox[1] = UINT_MAX;
|
||||
r->file_bbox[2] = r->file_bbox[3] = 0;
|
||||
@@ -1674,7 +1676,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
int n;
|
||||
|
||||
while ((n = fp->read(buf, READ_BUF)) > 0) {
|
||||
fwrite_check(buf, sizeof(char), n, readfp, reading.c_str());
|
||||
std::atomic<long long> readingpos;
|
||||
fwrite_check(buf, sizeof(char), n, readfp, &readingpos, reading.c_str());
|
||||
ahead += n;
|
||||
|
||||
if (buf[n - 1] == read_parallel_this && ahead > PARSE_MIN) {
|
||||
@@ -1803,13 +1806,10 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
if (!quiet) {
|
||||
fprintf(stderr, " \r");
|
||||
// (stderr, "Read 10000.00 million features\r", *progress_seq / 1000000.0);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (fclose(readers[i].metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
if (fclose(readers[i].geomfile) != 0) {
|
||||
perror("fclose geom");
|
||||
exit(EXIT_CLOSE);
|
||||
@@ -1824,21 +1824,16 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("stat geom\n");
|
||||
exit(EXIT_STAT);
|
||||
}
|
||||
if (fstat(readers[i].metafd, &readers[i].metast) != 0) {
|
||||
perror("stat meta\n");
|
||||
exit(EXIT_STAT);
|
||||
}
|
||||
}
|
||||
|
||||
// Create a combined string pool and a combined metadata file
|
||||
// Create a combined string pool
|
||||
// but keep track of the offsets into it since we still need
|
||||
// segment+offset to find the data.
|
||||
|
||||
// 2 * CPUS: One per input thread, one per tiling thread
|
||||
long long pool_off[2 * CPUS];
|
||||
long long meta_off[2 * CPUS];
|
||||
for (size_t i = 0; i < 2 * CPUS; i++) {
|
||||
pool_off[i] = meta_off[i] = 0;
|
||||
pool_off[i] = 0;
|
||||
}
|
||||
|
||||
char poolname[strlen(tmpdir) + strlen("/pool.XXXXXXXX") + 1];
|
||||
@@ -1858,60 +1853,51 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
|
||||
unlink(poolname);
|
||||
|
||||
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
|
||||
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
|
||||
|
||||
int metafd = mkstemp_cloexec(metaname);
|
||||
if (metafd < 0) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
FILE *metafile = fopen_oflag(metaname, "wb", O_WRONLY | O_CLOEXEC);
|
||||
if (metafile == NULL) {
|
||||
perror(metaname);
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
unlink(metaname);
|
||||
|
||||
std::atomic<long long> metapos(0);
|
||||
std::atomic<long long> poolpos(0);
|
||||
|
||||
for (size_t i = 0; i < CPUS; i++) {
|
||||
if (readers[i].metapos > 0) {
|
||||
void *map = mmap(NULL, readers[i].metapos, PROT_READ, MAP_PRIVATE, readers[i].metafd, 0);
|
||||
if (map == MAP_FAILED) {
|
||||
perror("mmap unmerged meta");
|
||||
// If the memfile is not done yet, it is in memory, so just copy the memory.
|
||||
// Otherwise, we need to merge memory and file.
|
||||
|
||||
if (readers[i].poolfile->fp == NULL) {
|
||||
// still in memory
|
||||
|
||||
if (readers[i].poolfile->map.size() > 0) {
|
||||
if (fwrite(readers[i].poolfile->map.c_str(), readers[i].poolfile->map.size(), 1, poolfile) != 1) {
|
||||
perror("Reunify string pool");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
}
|
||||
|
||||
pool_off[i] = poolpos;
|
||||
poolpos += readers[i].poolfile->map.size();
|
||||
} else {
|
||||
// split into memory and file
|
||||
|
||||
if (fflush(readers[i].poolfile->fp) != 0) {
|
||||
perror("fflush poolfile");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
|
||||
char *s = (char *) mmap(NULL, readers[i].poolfile->off, PROT_READ, MAP_PRIVATE, readers[i].poolfile->fd, 0);
|
||||
if (s == MAP_FAILED) {
|
||||
perror("mmap string pool for copy");
|
||||
exit(EXIT_MEMORY);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_SEQUENTIAL);
|
||||
madvise(map, readers[i].metapos, MADV_WILLNEED);
|
||||
if (fwrite(map, readers[i].metapos, 1, metafile) != 1) {
|
||||
perror("Reunify meta");
|
||||
madvise(s, readers[i].poolfile->off, MADV_SEQUENTIAL);
|
||||
if (fwrite(s, sizeof(char), readers[i].poolfile->off, poolfile) != readers[i].poolfile->off) {
|
||||
perror("Reunify string pool (split)");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
madvise(map, readers[i].metapos, MADV_DONTNEED);
|
||||
if (munmap(map, readers[i].metapos) != 0) {
|
||||
perror("unmap unmerged meta");
|
||||
if (munmap(s, readers[i].poolfile->off) != 0) {
|
||||
perror("unmap string pool for copy");
|
||||
exit(EXIT_MEMORY);
|
||||
}
|
||||
|
||||
pool_off[i] = poolpos;
|
||||
poolpos += readers[i].poolfile->off;
|
||||
}
|
||||
|
||||
meta_off[i] = metapos;
|
||||
metapos += readers[i].metapos;
|
||||
if (close(readers[i].metafd) != 0) {
|
||||
perror("close unmerged meta");
|
||||
}
|
||||
|
||||
if (readers[i].poolfile->off > 0) {
|
||||
if (fwrite(readers[i].poolfile->map, readers[i].poolfile->off, 1, poolfile) != 1) {
|
||||
perror("Reunify string pool");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
}
|
||||
|
||||
pool_off[i] = poolpos;
|
||||
poolpos += readers[i].poolfile->off;
|
||||
memfile_close(readers[i].poolfile);
|
||||
}
|
||||
|
||||
@@ -1919,17 +1905,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("fclose pool");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
if (fclose(metafile) != 0) {
|
||||
perror("fclose meta");
|
||||
exit(EXIT_CLOSE);
|
||||
}
|
||||
|
||||
char *meta = (char *) mmap(NULL, metapos, PROT_READ, MAP_PRIVATE, metafd, 0);
|
||||
if (meta == MAP_FAILED) {
|
||||
perror("mmap meta");
|
||||
exit(EXIT_MEMORY);
|
||||
}
|
||||
madvise(meta, metapos, MADV_RANDOM);
|
||||
|
||||
char *stringpool = NULL;
|
||||
if (poolpos > 0) { // Will be 0 if -X was specified
|
||||
@@ -1973,7 +1948,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
unlink(geomname);
|
||||
|
||||
unsigned iz = 0, ix = 0, iy = 0;
|
||||
choose_first_zoom(file_bbox, readers, &iz, &ix, &iy, minzoom, buffer);
|
||||
choose_first_zoom(file_bbox, file_bbox1, file_bbox2, readers, &iz, &ix, &iy, minzoom, buffer);
|
||||
|
||||
if (justx >= 0) {
|
||||
iz = minzoom;
|
||||
@@ -1983,7 +1958,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
|
||||
std::atomic<long long> geompos(0);
|
||||
|
||||
/* initial tile is 0/0/0 */
|
||||
/* initial tile is normally 0/0/0 but can be iz/ix/iy if limited to one tile */
|
||||
serialize_int(geomfile, iz, &geompos, fname);
|
||||
serialize_uint(geomfile, ix, &geompos, fname);
|
||||
serialize_uint(geomfile, iy, &geompos, fname);
|
||||
@@ -1991,7 +1966,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
radix(readers, CPUS, geomfile, indexfile, tmpdir, &geompos, maxzoom, basezoom, droprate, gamma);
|
||||
|
||||
/* end of tile */
|
||||
serialize_byte(geomfile, -2, &geompos, fname);
|
||||
serialize_ulong_long(geomfile, 0, &geompos, fname); // EOF
|
||||
|
||||
if (fclose(geomfile) != 0) {
|
||||
perror("fclose geom");
|
||||
@@ -2014,9 +1989,8 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
if (!quiet) {
|
||||
long long s = progress_seq;
|
||||
long long geompos_print = geompos;
|
||||
long long metapos_print = metapos;
|
||||
long long poolpos_print = poolpos;
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print);
|
||||
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of string pool\n", s, geompos_print, poolpos_print);
|
||||
}
|
||||
|
||||
if (indexpos == 0) {
|
||||
@@ -2083,6 +2057,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
progress = nprogress;
|
||||
if (!quiet && !quiet_progress && progress_time()) {
|
||||
fprintf(stderr, "Maxzoom: %lld%% \r", progress);
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2138,7 +2113,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
}
|
||||
|
||||
bool changed = false;
|
||||
while (maxzoom < 32 - full_detail && maxzoom < 33 - low_detail && cluster_distance > 0) {
|
||||
while (maxzoom < 32 - full_detail && maxzoom < 33 - low_detail && maxzoom < cluster_maxzoom && cluster_distance > 0) {
|
||||
unsigned long long zoom_mingap = ((1LL << (32 - maxzoom)) / 256 * cluster_distance) * ((1LL << (32 - maxzoom)) / 256 * cluster_distance);
|
||||
if (avg > zoom_mingap) {
|
||||
break;
|
||||
@@ -2148,7 +2123,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
changed = true;
|
||||
}
|
||||
if (changed) {
|
||||
printf("Choosing a maxzoom of -z%d to keep most features distinct with cluster distance %d\n", maxzoom, cluster_distance);
|
||||
printf("Choosing a maxzoom of -z%d to keep most features distinct with cluster distance %d and cluster maxzoom %d\n", maxzoom, cluster_distance, cluster_maxzoom);
|
||||
}
|
||||
|
||||
if (droprate == -3) {
|
||||
@@ -2239,6 +2214,11 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
}
|
||||
}
|
||||
|
||||
if (cluster_maxzoom >= maxzoom && guess_cluster_maxzoom) {
|
||||
cluster_maxzoom = maxzoom - 1;
|
||||
fprintf(stderr, "Choosing a cluster maxzoom of -k%d to make all features visible at maximum zoom %d\n", cluster_maxzoom, maxzoom);
|
||||
}
|
||||
|
||||
if (basezoom < 0 || droprate < 0) {
|
||||
struct tile {
|
||||
unsigned x;
|
||||
@@ -2269,6 +2249,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
progress = nprogress;
|
||||
if (!quiet && !quiet_progress && progress_time()) {
|
||||
fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress);
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2518,7 +2499,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
std::atomic<unsigned> midx(0);
|
||||
std::atomic<unsigned> midy(0);
|
||||
std::vector<strategy> strategies;
|
||||
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies);
|
||||
int written = traverse_zooms(fd, size, stringpool, &midx, &midy, maxzoom, minzoom, outdb, outdir, buffer, fname, tmpdir, gamma, full_detail, low_detail, min_detail, pool_off, initial_x, initial_y, simplification, maxzoom_simplification, layermaps, prefilter, postfilter, attribute_accum, filter, strategies, iz);
|
||||
|
||||
if (maxzoom != written) {
|
||||
if (written > minzoom) {
|
||||
@@ -2531,14 +2512,6 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
}
|
||||
}
|
||||
|
||||
madvise(meta, metapos, MADV_DONTNEED);
|
||||
if (munmap(meta, metapos) != 0) {
|
||||
perror("munmap meta");
|
||||
}
|
||||
if (close(metafd) < 0) {
|
||||
perror("close meta");
|
||||
}
|
||||
|
||||
if (poolpos > 0) {
|
||||
madvise((void *) stringpool, poolpos, MADV_DONTNEED);
|
||||
if (munmap(stringpool, poolpos) != 0) {
|
||||
@@ -2549,6 +2522,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
perror("close pool");
|
||||
}
|
||||
|
||||
// mbtiles-style bounding box and center
|
||||
double minlat = 0, minlon = 0, maxlat = 0, maxlon = 0, midlat = 0, midlon = 0;
|
||||
|
||||
tile2lonlat(midx, midy, maxzoom, &minlon, &maxlat);
|
||||
@@ -2573,6 +2547,17 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
midlon = maxlon;
|
||||
}
|
||||
|
||||
// antimeridian-aware bounding box
|
||||
double minlat2 = 0, minlon2 = 0, maxlat2 = 0, maxlon2 = 0;
|
||||
// choose whichever of the two calculated bboxes is narrower
|
||||
if (file_bbox2[2] - file_bbox2[0] < file_bbox1[2] - file_bbox1[0]) {
|
||||
tile2lonlat(file_bbox2[0], file_bbox2[1], 32, &minlon2, &maxlat2);
|
||||
tile2lonlat(file_bbox2[2], file_bbox2[3], 32, &maxlon2, &minlat2);
|
||||
} else {
|
||||
tile2lonlat(file_bbox1[0], file_bbox1[1], 32, &minlon2, &maxlat2);
|
||||
tile2lonlat(file_bbox1[2], file_bbox1[3], 32, &maxlon2, &minlat2);
|
||||
}
|
||||
|
||||
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
|
||||
|
||||
for (auto ai = merged_lm.begin(); ai != merged_lm.end(); ++ai) {
|
||||
@@ -2580,7 +2565,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
|
||||
ai->second.maxzoom = maxzoom;
|
||||
}
|
||||
|
||||
metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies);
|
||||
metadata m = make_metadata(fname, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, minlat2, minlon2, maxlat2, maxlon2, midlat, midlon, attribution, merged_lm, true, description, !prevent[P_TILE_STATS], attribute_descriptions, "tippecanoe", commandline, strategies);
|
||||
if (outdb != NULL) {
|
||||
mbtiles_write_metadata(outdb, m, forcetable);
|
||||
} else {
|
||||
@@ -2735,6 +2720,7 @@ int main(int argc, char **argv) {
|
||||
const char *postfilter = NULL;
|
||||
bool guess_maxzoom = false;
|
||||
int minimum_maxzoom = 0;
|
||||
bool guess_cluster_maxzoom = false;
|
||||
|
||||
std::set<std::string> exclude, include;
|
||||
std::map<std::string, int> attribute_types;
|
||||
@@ -2745,6 +2731,8 @@ int main(int argc, char **argv) {
|
||||
int files_open_at_start;
|
||||
json_object *filter = NULL;
|
||||
|
||||
memsize = calc_memsize();
|
||||
|
||||
for (i = 0; i < 256; i++) {
|
||||
prevent[i] = 0;
|
||||
additional[i] = 0;
|
||||
@@ -2811,6 +2799,7 @@ int main(int argc, char **argv) {
|
||||
{"drop-lines", no_argument, &additional[A_LINE_DROP], 1},
|
||||
{"drop-polygons", no_argument, &additional[A_POLYGON_DROP], 1},
|
||||
{"cluster-distance", required_argument, 0, 'K'},
|
||||
{"cluster-maxzoom", required_argument, 0, 'k'},
|
||||
|
||||
{"Dropping or merging a fraction of features to keep under tile size limits", 0, 0, 0},
|
||||
{"drop-densest-as-needed", no_argument, &additional[A_DROP_DENSEST_AS_NEEDED], 1},
|
||||
@@ -3114,6 +3103,15 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
break;
|
||||
|
||||
case 'k':
|
||||
if (strcmp(optarg, "g") == 0) {
|
||||
cluster_maxzoom = MAX_ZOOM - 1;
|
||||
guess_cluster_maxzoom = true;
|
||||
} else {
|
||||
cluster_maxzoom = atoi_require(optarg, "Cluster maxzoom");
|
||||
}
|
||||
break;
|
||||
|
||||
case 'd':
|
||||
full_detail = atoi_require(optarg, "Full detail");
|
||||
if (full_detail > 30) {
|
||||
@@ -3523,9 +3521,12 @@ int main(int argc, char **argv) {
|
||||
|
||||
long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
|
||||
|
||||
long long file_bbox1[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0}; // standard -180 to 180 world plane
|
||||
long long file_bbox2[4] = {0x1FFFFFFFF, 0xFFFFFFFF, 0x100000000, 0}; // 0 to 360 world plane
|
||||
|
||||
auto input_ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles
|
||||
: out_dir,
|
||||
maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, prefilter, postfilter, description, guess_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom);
|
||||
maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_dir, &exclude, &include, exclude_all, filter, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, file_bbox1, file_bbox2, prefilter, postfilter, description, guess_maxzoom, guess_cluster_maxzoom, &attribute_types, argv[0], &attribute_accum, attribute_descriptions, commandline, minimum_maxzoom);
|
||||
|
||||
ret = std::get<0>(input_ret);
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <stddef.h>
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "json_logger.hpp"
|
||||
|
||||
@@ -47,10 +48,12 @@ extern int extra_detail;
|
||||
|
||||
extern size_t CPUS;
|
||||
extern size_t TEMP_FILES;
|
||||
extern size_t memsize;
|
||||
|
||||
extern size_t max_tile_size;
|
||||
extern size_t max_tile_features;
|
||||
extern int cluster_distance;
|
||||
extern int cluster_maxzoom;
|
||||
extern std::string attribute_for_id;
|
||||
extern int tiny_polygon_size;
|
||||
extern size_t limit_tile_feature_count;
|
||||
|
||||
+7
-1
@@ -395,7 +395,7 @@ Parallel processing will also be automatic if the input file is in FlatGeobuf fo
|
||||
.IP \(bu 2
|
||||
\fB\fC\-zg\fR or \fB\fC\-\-maximum\-zoom=g\fR: Guess what is probably a reasonable maxzoom based on the spacing of features.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-\-smallest\-maximum\-zoom\-guess=\fR\fIzoom\fP: Guess what is probably a reasonable maxzoom based on the spacing of features, but using the specified \fIzoom\fP if a lower maxzoom is guessed.
|
||||
\fB\fC\-\-smallest\-maximum\-zoom\-guess=\fR\fIzoom\fP: Guess what is probably a reasonable maxzoom based on the spacing of features, but using the specified \fIzoom\fP if a lower maxzoom is guessed. If \fB\fC\-Bg\fR is also set, the base zoom will be set to the guessed maxzoom, with all the points carried forward into additional zooms through the one specified.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-Z\fR \fIzoom\fP or \fB\fC\-\-minimum\-zoom=\fR\fIzoom\fP: Minzoom: the lowest zoom level for which tiles are generated (default 0)
|
||||
.IP \(bu 2
|
||||
@@ -573,6 +573,10 @@ preference to retaining points in sparse areas and dropping points in dense area
|
||||
\fB\fC\-ap\fR or \fB\fC\-\-drop\-polygons\fR: Let "dot" dropping at lower zooms apply to polygons too
|
||||
.IP \(bu 2
|
||||
\fB\fC\-K\fR \fIdistance\fP or \fB\fC\-\-cluster\-distance=\fR\fIdistance\fP: Cluster points (as with \fB\fC\-\-cluster\-densest\-as\-needed\fR, but without the experimental discovery process) that are approximately within \fIdistance\fP of each other. The units are tile coordinates within a nominally 256\-pixel tile, so the maximum value of 255 allows only one feature per tile. Values around 10 are probably appropriate for typical marker sizes. See \fB\fC\-\-cluster\-densest\-as\-needed\fR below for behavior.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-k\fR \fIzoom\fP or \fB\fC\-\-cluster\-maxzoom=\fR\fIzoom\fP: Max zoom on which to cluster points if clustering is enabled.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-kg\fR or \fB\fC\-\-cluster\-maxzoom=g\fR: Set \fB\fC\-\-cluster\-maxzoom=\fR to \fB\fCmaxzoom \- 1\fR so that all features are visible at the maximum zoom level.
|
||||
.RE
|
||||
.SS Dropping a fraction of features to keep under tile size limits
|
||||
.RS
|
||||
@@ -952,6 +956,8 @@ The options are:
|
||||
\fB\fC\-e\fR \fIdirectory\fP or \fB\fC\-\-output\-to\-directory=\fR\fIdirectory\fP: Write the new tiles to the specified directory instead of to an mbtiles file.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-f\fR or \fB\fC\-\-force\fR: Remove \fIout.mbtiles\fP if it already exists.
|
||||
.IP \(bu 2
|
||||
\fB\fC\-r\fR or \fB\fC\-\-read\-from\fR: list of input mbtiles to read from.
|
||||
.RE
|
||||
.SS Tileset description and attribution
|
||||
.RS
|
||||
|
||||
+15
-1
@@ -530,6 +530,15 @@ void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) {
|
||||
}
|
||||
sqlite3_free(sql);
|
||||
|
||||
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('antimeridian_adjusted_bounds', '%f,%f,%f,%f');", m.minlon2, m.minlat2, m.maxlon2, m.maxlat2);
|
||||
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
|
||||
fprintf(stderr, "set bounds: %s\n", err);
|
||||
if (!forcetable) {
|
||||
exit(EXIT_SQLITE);
|
||||
}
|
||||
}
|
||||
sqlite3_free(sql);
|
||||
|
||||
sql = sqlite3_mprintf("INSERT INTO metadata (name, value) VALUES ('type', %Q);", m.type.c_str());
|
||||
if (sqlite3_exec(db, sql, NULL, NULL, &err) != SQLITE_OK) {
|
||||
fprintf(stderr, "set type: %s\n", err);
|
||||
@@ -629,7 +638,7 @@ void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable) {
|
||||
}
|
||||
}
|
||||
|
||||
metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies) {
|
||||
metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies) {
|
||||
metadata m;
|
||||
|
||||
m.name = fname;
|
||||
@@ -646,6 +655,11 @@ metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minla
|
||||
m.maxlat = maxlat;
|
||||
m.maxlon = maxlon;
|
||||
|
||||
m.minlat2 = minlat2;
|
||||
m.minlon2 = minlon2;
|
||||
m.maxlat2 = maxlat2;
|
||||
m.maxlon2 = maxlon2;
|
||||
|
||||
m.center_lat = midlat;
|
||||
m.center_lon = midlon;
|
||||
m.center_z = maxzoom;
|
||||
|
||||
+2
-1
@@ -52,6 +52,7 @@ struct metadata {
|
||||
int maxzoom;
|
||||
|
||||
double minlat, minlon, maxlat, maxlon;
|
||||
double minlat2, minlon2, maxlat2, maxlon2; // antimeridian-aware
|
||||
|
||||
double center_lon, center_lat;
|
||||
int center_z;
|
||||
@@ -74,7 +75,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
|
||||
void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
|
||||
void mbtiles_erase_zoom(sqlite3 *outdb, int z);
|
||||
|
||||
metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);
|
||||
metadata make_metadata(const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double minlat2, double minlon2, double maxlat2, double maxlon2, double midlat, double midlon, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);
|
||||
void mbtiles_write_metadata(sqlite3 *db, const metadata &m, bool forcetable);
|
||||
|
||||
void mbtiles_close(sqlite3 *outdb, const char *pgm);
|
||||
|
||||
+46
-29
@@ -12,33 +12,34 @@ struct memfile *memfile_open(int fd) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *map = (char *) mmap(NULL, INITIAL, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if (map == MAP_FAILED) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct memfile *mf = new memfile;
|
||||
if (mf == NULL) {
|
||||
munmap(map, INITIAL);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
mf->fd = fd;
|
||||
mf->map = map;
|
||||
mf->len = INITIAL;
|
||||
mf->off = 0;
|
||||
mf->tree = 0;
|
||||
mf->off = 0;
|
||||
|
||||
return mf;
|
||||
}
|
||||
|
||||
int memfile_close(struct memfile *file) {
|
||||
if (munmap(file->map, file->len) != 0) {
|
||||
return -1;
|
||||
}
|
||||
// If it isn't full yet, flush out the string to the file now.
|
||||
// If it is full, close out the buffered file writer.
|
||||
|
||||
if (file->fd >= 0) {
|
||||
if (close(file->fd) != 0) {
|
||||
if (file->fp == NULL) {
|
||||
if (write(file->fd, file->map.c_str(), file->map.size()) != (ssize_t) file->map.size()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (file->fd >= 0) {
|
||||
if (close(file->fd) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (fclose(file->fp) != 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@@ -48,24 +49,40 @@ int memfile_close(struct memfile *file) {
|
||||
}
|
||||
|
||||
int memfile_write(struct memfile *file, void *s, long long len) {
|
||||
if (file->off + len > file->len) {
|
||||
if (munmap(file->map, file->len) != 0) {
|
||||
return -1;
|
||||
}
|
||||
// If it is full, append to the file.
|
||||
// If it is not full yet, append to the string in memory.
|
||||
|
||||
file->len += (len + INCREMENT + 1) / INCREMENT * INCREMENT;
|
||||
|
||||
if (ftruncate(file->fd, file->len) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
file->map = (char *) mmap(NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0);
|
||||
if (file->map == MAP_FAILED) {
|
||||
return -1;
|
||||
if (file->fp != NULL) {
|
||||
if (fwrite(s, sizeof(char), len, file->fp) != (size_t) len) {
|
||||
return 0;
|
||||
}
|
||||
file->off += len;
|
||||
} else {
|
||||
file->map.append(std::string((char *) s, len));
|
||||
file->off += len;
|
||||
}
|
||||
|
||||
memcpy(file->map + file->off, s, len);
|
||||
file->off += len;
|
||||
return len;
|
||||
}
|
||||
|
||||
void memfile_full(struct memfile *file) {
|
||||
// The file is full. Write out a copy of whatever has accumulated in memory
|
||||
// to the file, and switch to appending to the file. Existing references
|
||||
// into the memory still work.
|
||||
|
||||
if (file->fp != NULL) {
|
||||
fprintf(stderr, "memfile marked full twice\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
file->fp = fdopen(file->fd, "wb");
|
||||
if (file->fp == NULL) {
|
||||
fprintf(stderr, "fdopen memfile: %s\n", strerror(errno));
|
||||
exit(EXIT_OPEN);
|
||||
}
|
||||
|
||||
if (fwrite(file->map.c_str(), sizeof(char), file->map.size(), file->fp) != file->map.size()) {
|
||||
fprintf(stderr, "memfile write: %s\n", strerror(errno));
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
}
|
||||
|
||||
+6
-7
@@ -2,21 +2,20 @@
|
||||
#define MEMFILE_HPP
|
||||
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include "errors.hpp"
|
||||
|
||||
struct memfile {
|
||||
int fd = 0;
|
||||
char *map = NULL;
|
||||
std::atomic<long long> len;
|
||||
long long off = 0;
|
||||
std::string map;
|
||||
unsigned long tree = 0;
|
||||
|
||||
memfile()
|
||||
: len(0) {
|
||||
}
|
||||
FILE *fp = NULL;
|
||||
size_t off = 0;
|
||||
};
|
||||
|
||||
struct memfile *memfile_open(int fd);
|
||||
int memfile_close(struct memfile *file);
|
||||
int memfile_write(struct memfile *file, void *s, long long len);
|
||||
void memfile_full(struct memfile *file);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -82,14 +82,14 @@ int decompress(std::string const &input, std::string &output) {
|
||||
}
|
||||
|
||||
// https://github.com/mapbox/mapnik-vector-tile/blob/master/src/vector_tile_compression.hpp
|
||||
int compress(std::string const &input, std::string &output) {
|
||||
int compress(std::string const &input, std::string &output, bool gz) {
|
||||
z_stream deflate_s;
|
||||
deflate_s.zalloc = Z_NULL;
|
||||
deflate_s.zfree = Z_NULL;
|
||||
deflate_s.opaque = Z_NULL;
|
||||
deflate_s.avail_in = 0;
|
||||
deflate_s.next_in = Z_NULL;
|
||||
deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
|
||||
deflateInit2(&deflate_s, Z_BEST_COMPRESSION, Z_DEFLATED, gz ? 31 : 15, 8, Z_DEFAULT_STRATEGY);
|
||||
deflate_s.next_in = (Bytef *) input.data();
|
||||
deflate_s.avail_in = input.size();
|
||||
size_t length = 0;
|
||||
|
||||
@@ -115,7 +115,7 @@ struct mvt_tile {
|
||||
|
||||
bool is_compressed(std::string const &data);
|
||||
int decompress(std::string const &input, std::string &output);
|
||||
int compress(std::string const &input, std::string &output);
|
||||
int compress(std::string const &input, std::string &output, bool gz);
|
||||
int dezig(unsigned n);
|
||||
|
||||
mvt_value stringified_to_mvt_value(int type, const char *s);
|
||||
|
||||
@@ -405,7 +405,6 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
|
||||
sf.bbox[0] = sf.bbox[1] = LLONG_MAX;
|
||||
sf.bbox[2] = sf.bbox[3] = LLONG_MIN;
|
||||
sf.extent = 0;
|
||||
sf.metapos = 0;
|
||||
sf.has_id = false;
|
||||
|
||||
std::string layername = "unknown";
|
||||
|
||||
+15
-22
@@ -183,18 +183,6 @@ struct entry_zxy {
|
||||
}
|
||||
};
|
||||
|
||||
struct {
|
||||
bool operator()(entry_zxy a, entry_zxy b) const {
|
||||
if (a.z != b.z) {
|
||||
return a.z < b.z;
|
||||
}
|
||||
if (a.x != b.x) {
|
||||
return a.x < b.x;
|
||||
}
|
||||
return a.y < b.y;
|
||||
}
|
||||
} entry_zxy_cmp;
|
||||
|
||||
struct varint_too_long_exception : std::exception {
|
||||
const char *what() const noexcept override {
|
||||
return "varint too long exception";
|
||||
@@ -311,14 +299,14 @@ void rotate(int64_t n, int64_t &x, int64_t &y, int64_t rx, int64_t ry) {
|
||||
}
|
||||
|
||||
zxy t_on_level(uint8_t z, uint64_t pos) {
|
||||
int64_t n = 1 << z;
|
||||
int64_t n = 1LL << z;
|
||||
int64_t rx, ry, s, t = pos;
|
||||
int64_t tx = 0;
|
||||
int64_t ty = 0;
|
||||
|
||||
for (s = 1; s < n; s *= 2) {
|
||||
rx = 1 & (t / 2);
|
||||
ry = 1 & (t ^ rx);
|
||||
rx = 1LL & (t / 2);
|
||||
ry = 1LL & (t ^ rx);
|
||||
rotate(s, tx, ty, rx, ry);
|
||||
tx += s * rx;
|
||||
ty += s * ry;
|
||||
@@ -385,28 +373,33 @@ entryv3 find_tile(const std::vector<entryv3> &entries, uint64_t tile_id) {
|
||||
|
||||
inline zxy tileid_to_zxy(uint64_t tileid) {
|
||||
uint64_t acc = 0;
|
||||
uint8_t t_z = 0;
|
||||
while (true) {
|
||||
for (uint8_t t_z = 0; t_z < 32; t_z++) {
|
||||
uint64_t num_tiles = (1LL << t_z) * (1LL << t_z);
|
||||
if (acc + num_tiles > tileid) {
|
||||
return t_on_level(t_z, tileid - acc);
|
||||
}
|
||||
acc += num_tiles;
|
||||
t_z++;
|
||||
}
|
||||
throw std::overflow_error("tile zoom exceeds 64-bit limit");
|
||||
}
|
||||
|
||||
inline uint64_t zxy_to_tileid(uint8_t z, uint32_t x, uint32_t y) {
|
||||
if (z > 31) {
|
||||
throw std::overflow_error("tile zoom exceeds 64-bit limit");
|
||||
}
|
||||
if (x > (1 << z) - 1 || y > (1 << z) - 1) {
|
||||
throw std::overflow_error("tile x/y outside zoom level bounds");
|
||||
}
|
||||
uint64_t acc = 0;
|
||||
for (uint8_t t_z = 0; t_z < z; t_z++) acc += (1LL << t_z) * (1LL << t_z);
|
||||
int64_t n = 1 << z;
|
||||
int64_t n = 1LL << z;
|
||||
int64_t rx, ry, s, d = 0;
|
||||
int64_t tx = x;
|
||||
int64_t ty = y;
|
||||
for (s = n / 2; s > 0; s /= 2) {
|
||||
rx = (tx & s) > 0;
|
||||
ry = (ty & s) > 0;
|
||||
d += s * s * ((3 * rx) ^ ry);
|
||||
d += s * s * ((3LL * rx) ^ ry);
|
||||
rotate(s, tx, ty, rx, ry);
|
||||
}
|
||||
return acc + d;
|
||||
@@ -491,7 +484,7 @@ inline std::tuple<std::string, std::string, int> build_root_leaves(const std::fu
|
||||
std::vector<pmtiles::entryv3> root_entries;
|
||||
std::string leaves_bytes;
|
||||
int num_leaves = 0;
|
||||
for (size_t i = 0; i <= entries.size(); i += leaf_size) {
|
||||
for (size_t i = 0; i < entries.size(); i += leaf_size) {
|
||||
num_leaves++;
|
||||
int end = i + leaf_size;
|
||||
if (i + leaf_size > entries.size()) {
|
||||
@@ -585,4 +578,4 @@ inline std::pair<uint64_t, uint32_t> get_tile(const std::function<std::string(co
|
||||
}
|
||||
|
||||
} // namespace pmtiles
|
||||
#endif
|
||||
#endif
|
||||
|
||||
+6
-2
@@ -57,7 +57,7 @@ std::string compress_fn(const std::string &input, uint8_t compression) {
|
||||
if (compression == pmtiles::COMPRESSION_NONE) {
|
||||
output = input;
|
||||
} else if (compression == pmtiles::COMPRESSION_GZIP) {
|
||||
compress(input, output);
|
||||
compress(input, output, true);
|
||||
} else {
|
||||
throw std::runtime_error("Unknown or unsupported compression.");
|
||||
}
|
||||
@@ -105,6 +105,9 @@ std::string metadata_to_pmtiles_json(metadata m) {
|
||||
out(state, "generator", m.generator);
|
||||
out(state, "generator_options", m.generator_options);
|
||||
|
||||
std::string bounds2 = std::to_string(m.minlon2) + "," + std::to_string(m.minlat2) + "," + std::to_string(m.maxlon2) + "," + std::to_string(m.maxlat2);
|
||||
out(state, "antimeridian_adjusted_bounds", bounds2);
|
||||
|
||||
if (m.vector_layers_json.size() > 0) {
|
||||
state.json_comma_newline();
|
||||
state.json_write_string("vector_layers");
|
||||
@@ -121,7 +124,7 @@ std::string metadata_to_pmtiles_json(metadata m) {
|
||||
state.json_end_hash();
|
||||
state.json_write_newline();
|
||||
std::string compressed;
|
||||
compress(buf, compressed);
|
||||
compress(buf, compressed, true);
|
||||
return compressed;
|
||||
}
|
||||
|
||||
@@ -200,6 +203,7 @@ void mbtiles_map_image_to_pmtiles(char *fname, metadata m, bool tile_compression
|
||||
pmtiles::zxy zxy = pmtiles::tileid_to_zxy(tile_id);
|
||||
if (!quiet && !quiet_progress) {
|
||||
fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, zxy.z, zxy.x, zxy.y);
|
||||
fflush(stderr);
|
||||
}
|
||||
sqlite3_bind_int(map_stmt, 1, zxy.z);
|
||||
sqlite3_bind_int(map_stmt, 2, zxy.x);
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
#include <math.h>
|
||||
#include "main.hpp"
|
||||
#include "memfile.hpp"
|
||||
#include "pool.hpp"
|
||||
#include "errors.hpp"
|
||||
@@ -42,24 +43,26 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
|
||||
}
|
||||
|
||||
while (*sp != 0) {
|
||||
int cmp = swizzlecmp(s, poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off + 1);
|
||||
int cmp = swizzlecmp(s, poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off + 1);
|
||||
|
||||
if (cmp == 0) {
|
||||
cmp = type - (poolfile->map + ((struct stringpool *) (treefile->map + *sp))->off)[0];
|
||||
cmp = type - (poolfile->map.c_str() + ((struct stringpool *) (treefile->map.c_str() + *sp))->off)[0];
|
||||
}
|
||||
|
||||
if (cmp < 0) {
|
||||
sp = &(((struct stringpool *) (treefile->map + *sp))->left);
|
||||
sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->left);
|
||||
} else if (cmp > 0) {
|
||||
sp = &(((struct stringpool *) (treefile->map + *sp))->right);
|
||||
sp = &(((struct stringpool *) (treefile->map.c_str() + *sp))->right);
|
||||
} else {
|
||||
return ((struct stringpool *) (treefile->map + *sp))->off;
|
||||
return ((struct stringpool *) (treefile->map.c_str() + *sp))->off;
|
||||
}
|
||||
|
||||
depth++;
|
||||
if (depth > max) {
|
||||
// Search is very deep, so string is probably unique.
|
||||
// Add it to the pool without adding it to the search tree.
|
||||
// This might go either to memory or the file, depending on whether
|
||||
// the pool is full yet.
|
||||
|
||||
long long off = poolfile->off;
|
||||
if (memfile_write(poolfile, &type, 1) < 0) {
|
||||
@@ -74,12 +77,41 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
|
||||
}
|
||||
}
|
||||
|
||||
// Size of memory divided by 10 from observation of OOM errors (when supposedly
|
||||
// 20% of memory is full) and onset of thrashing (when supposedly 15% of memory
|
||||
// is full) on ECS.
|
||||
if ((size_t) (poolfile->off + treefile->off) > memsize / CPUS / 10) {
|
||||
// If the pool and search tree get to be larger than physical memory,
|
||||
// then searching will start thrashing. Switch to appending strings
|
||||
// to the file instead of keeping them in memory.
|
||||
|
||||
if (poolfile->fp == NULL) {
|
||||
memfile_full(poolfile);
|
||||
}
|
||||
}
|
||||
|
||||
if (poolfile->fp != NULL) {
|
||||
// We are now appending to the file, so don't try to keep tree references
|
||||
// to the newly-added strings.
|
||||
|
||||
long long off = poolfile->off;
|
||||
if (memfile_write(poolfile, &type, 1) < 0) {
|
||||
perror("memfile write");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
if (memfile_write(poolfile, (void *) s, strlen(s) + 1) < 0) {
|
||||
perror("memfile write");
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
return off;
|
||||
}
|
||||
|
||||
// *sp is probably in the memory-mapped file, and will move if the file grows.
|
||||
long long ssp;
|
||||
if (sp == &treefile->tree) {
|
||||
ssp = -1;
|
||||
} else {
|
||||
ssp = ((char *) sp) - treefile->map;
|
||||
ssp = ((char *) sp) - treefile->map.c_str();
|
||||
}
|
||||
|
||||
long long off = poolfile->off;
|
||||
@@ -116,7 +148,7 @@ long long addpool(struct memfile *poolfile, struct memfile *treefile, const char
|
||||
if (ssp == -1) {
|
||||
treefile->tree = p;
|
||||
} else {
|
||||
*((long long *) (treefile->map + ssp)) = p;
|
||||
*((long long *) (treefile->map.c_str() + ssp)) = p;
|
||||
}
|
||||
return off;
|
||||
}
|
||||
|
||||
+4
-4
@@ -22,8 +22,8 @@ struct projection *projection = &projections[0];
|
||||
void lonlat2tile(double lon, double lat, int zoom, long long *x, long long *y) {
|
||||
// Place infinite and NaN coordinates off the edge of the Mercator plane
|
||||
|
||||
int lat_class = fpclassify(lat);
|
||||
int lon_class = fpclassify(lon);
|
||||
int lat_class = std::fpclassify(lat);
|
||||
int lon_class = std::fpclassify(lon);
|
||||
bool bad_lon = false;
|
||||
|
||||
if (lat_class == FP_INFINITE || lat_class == FP_NAN) {
|
||||
@@ -74,8 +74,8 @@ void tile2lonlat(long long x, long long y, int zoom, double *lon, double *lat) {
|
||||
void epsg3857totile(double ix, double iy, int zoom, long long *x, long long *y) {
|
||||
// Place infinite and NaN coordinates off the edge of the Mercator plane
|
||||
|
||||
int iy_class = fpclassify(iy);
|
||||
int ix_class = fpclassify(ix);
|
||||
int iy_class = std::fpclassify(iy);
|
||||
int ix_class = std::fpclassify(ix);
|
||||
|
||||
if (iy_class == FP_INFINITE || iy_class == FP_NAN) {
|
||||
iy = 40000000.0;
|
||||
|
||||
+147
-177
@@ -9,9 +9,11 @@
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
#include <limits.h>
|
||||
#include <zlib.h>
|
||||
#include "protozero/varint.hpp"
|
||||
#include "geometry.hpp"
|
||||
#include "mbtiles.hpp"
|
||||
#include "mvt.hpp"
|
||||
#include "tile.hpp"
|
||||
#include "serial.hpp"
|
||||
#include "options.hpp"
|
||||
@@ -24,15 +26,18 @@
|
||||
|
||||
// Offset coordinates to keep them positive
|
||||
#define COORD_OFFSET (4LL << 32)
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double)(a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_RIGHT(a) ((long long) std::round((double) (a) / (1LL << geometry_scale)))
|
||||
#define SHIFT_LEFT(a) ((((a) + (COORD_OFFSET >> geometry_scale)) << geometry_scale) - COORD_OFFSET)
|
||||
|
||||
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) {
|
||||
// write to file
|
||||
|
||||
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, std::atomic<long long> *fpos, const char *fname) {
|
||||
size_t w = fwrite(ptr, size, nitems, stream);
|
||||
if (w != nitems) {
|
||||
fprintf(stderr, "%s: Write to temporary file failed: %s\n", fname, strerror(errno));
|
||||
exit(EXIT_WRITE);
|
||||
}
|
||||
*fpos += size * nitems;
|
||||
return w;
|
||||
}
|
||||
|
||||
@@ -69,15 +74,54 @@ void serialize_ulong_long(FILE *out, unsigned long long zigzag, std::atomic<long
|
||||
}
|
||||
|
||||
void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname) {
|
||||
fwrite_check(&n, sizeof(signed char), 1, out, fname);
|
||||
*fpos += sizeof(signed char);
|
||||
fwrite_check(&n, sizeof(signed char), 1, out, fpos, fname);
|
||||
}
|
||||
|
||||
void serialize_uint(FILE *out, unsigned n, std::atomic<long long> *fpos, const char *fname) {
|
||||
fwrite_check(&n, sizeof(unsigned), 1, out, fname);
|
||||
*fpos += sizeof(unsigned);
|
||||
serialize_ulong_long(out, n, fpos, fname);
|
||||
}
|
||||
|
||||
// write to memory
|
||||
|
||||
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, std::string &stream) {
|
||||
stream += std::string((char *) ptr, size * nitems);
|
||||
return nitems;
|
||||
}
|
||||
|
||||
void serialize_ulong_long(std::string &out, unsigned long long zigzag) {
|
||||
while (1) {
|
||||
unsigned char b = zigzag & 0x7F;
|
||||
if ((zigzag >> 7) != 0) {
|
||||
b |= 0x80;
|
||||
out += b;
|
||||
zigzag >>= 7;
|
||||
} else {
|
||||
out += b;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void serialize_long_long(std::string &out, long long n) {
|
||||
unsigned long long zigzag = protozero::encode_zigzag64(n);
|
||||
|
||||
serialize_ulong_long(out, zigzag);
|
||||
}
|
||||
|
||||
void serialize_int(std::string &out, int n) {
|
||||
serialize_long_long(out, n);
|
||||
}
|
||||
|
||||
void serialize_byte(std::string &out, signed char n) {
|
||||
out += n;
|
||||
}
|
||||
|
||||
void serialize_uint(std::string &out, unsigned n) {
|
||||
serialize_ulong_long(out, n);
|
||||
}
|
||||
|
||||
// read from memory
|
||||
|
||||
void deserialize_int(char **f, int *n) {
|
||||
long long ll;
|
||||
deserialize_long_long(f, &ll);
|
||||
@@ -109,8 +153,9 @@ void deserialize_ulong_long(char **f, unsigned long long *zigzag) {
|
||||
}
|
||||
|
||||
void deserialize_uint(char **f, unsigned *n) {
|
||||
memcpy(n, *f, sizeof(unsigned));
|
||||
*f += sizeof(unsigned);
|
||||
unsigned long long v;
|
||||
deserialize_ulong_long(f, &v);
|
||||
*n = v;
|
||||
}
|
||||
|
||||
void deserialize_byte(char **f, signed char *n) {
|
||||
@@ -118,79 +163,26 @@ void deserialize_byte(char **f, signed char *n) {
|
||||
*f += sizeof(signed char);
|
||||
}
|
||||
|
||||
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos) {
|
||||
unsigned long long zigzag = 0;
|
||||
int ret = deserialize_ulong_long_io(f, &zigzag, geompos);
|
||||
*n = protozero::decode_zigzag64(zigzag);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_ulong_long_io(FILE *f, unsigned long long *zigzag, std::atomic<long long> *geompos) {
|
||||
*zigzag = 0;
|
||||
int shift = 0;
|
||||
|
||||
while (1) {
|
||||
int c = getc(f);
|
||||
if (c == EOF) {
|
||||
return 0;
|
||||
}
|
||||
(*geompos)++;
|
||||
|
||||
if ((c & 0x80) == 0) {
|
||||
*zigzag |= ((unsigned long long) c) << shift;
|
||||
shift += 7;
|
||||
break;
|
||||
} else {
|
||||
*zigzag |= ((unsigned long long) (c & 0x7F)) << shift;
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos) {
|
||||
long long ll = 0;
|
||||
int ret = deserialize_long_long_io(f, &ll, geompos);
|
||||
*n = ll;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos) {
|
||||
if (fread(n, sizeof(unsigned), 1, f) != 1) {
|
||||
return 0;
|
||||
}
|
||||
*geompos += sizeof(unsigned);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos) {
|
||||
int c = getc(f);
|
||||
if (c == EOF) {
|
||||
return 0;
|
||||
}
|
||||
*n = c;
|
||||
(*geompos)++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void write_geometry(drawvec const &dv, std::atomic<long long> *fpos, FILE *out, const char *fname, long long wx, long long wy) {
|
||||
static void write_geometry(drawvec const &dv, std::string &out, long long wx, long long wy) {
|
||||
for (size_t i = 0; i < dv.size(); i++) {
|
||||
if (dv[i].op == VT_MOVETO || dv[i].op == VT_LINETO) {
|
||||
serialize_byte(out, dv[i].op, fpos, fname);
|
||||
serialize_long_long(out, dv[i].x - wx, fpos, fname);
|
||||
serialize_long_long(out, dv[i].y - wy, fpos, fname);
|
||||
serialize_byte(out, dv[i].op);
|
||||
serialize_long_long(out, dv[i].x - wx);
|
||||
serialize_long_long(out, dv[i].y - wy);
|
||||
wx = dv[i].x;
|
||||
wy = dv[i].y;
|
||||
} else {
|
||||
serialize_byte(out, dv[i].op, fpos, fname);
|
||||
serialize_byte(out, dv[i].op);
|
||||
}
|
||||
}
|
||||
serialize_byte(out, VT_END);
|
||||
}
|
||||
|
||||
// called from generating the next zoom level
|
||||
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom) {
|
||||
serialize_byte(geomfile, sf->t, geompos, fname);
|
||||
std::string serialize_feature(serial_feature *sf, long long wx, long long wy) {
|
||||
std::string s;
|
||||
|
||||
serialize_byte(s, sf->t);
|
||||
|
||||
#define FLAG_LAYER 7
|
||||
|
||||
@@ -212,63 +204,56 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long
|
||||
layer |= sf->has_tippecanoe_minzoom << FLAG_MINZOOM;
|
||||
layer |= sf->has_tippecanoe_maxzoom << FLAG_MAXZOOM;
|
||||
|
||||
serialize_long_long(geomfile, layer, geompos, fname);
|
||||
serialize_long_long(s, layer);
|
||||
if (sf->seq != 0) {
|
||||
serialize_long_long(geomfile, sf->seq, geompos, fname);
|
||||
serialize_long_long(s, sf->seq);
|
||||
}
|
||||
if (sf->has_tippecanoe_minzoom) {
|
||||
serialize_int(geomfile, sf->tippecanoe_minzoom, geompos, fname);
|
||||
serialize_int(s, sf->tippecanoe_minzoom);
|
||||
}
|
||||
if (sf->has_tippecanoe_maxzoom) {
|
||||
serialize_int(geomfile, sf->tippecanoe_maxzoom, geompos, fname);
|
||||
serialize_int(s, sf->tippecanoe_maxzoom);
|
||||
}
|
||||
if (sf->has_id) {
|
||||
serialize_ulong_long(geomfile, sf->id, geompos, fname);
|
||||
serialize_ulong_long(s, sf->id);
|
||||
}
|
||||
|
||||
serialize_int(geomfile, sf->segment, geompos, fname);
|
||||
serialize_int(s, sf->segment);
|
||||
|
||||
write_geometry(sf->geometry, s, wx, wy);
|
||||
|
||||
write_geometry(sf->geometry, geompos, geomfile, fname, wx, wy);
|
||||
serialize_byte(geomfile, VT_END, geompos, fname);
|
||||
if (sf->index != 0) {
|
||||
serialize_ulong_long(geomfile, sf->index, geompos, fname);
|
||||
serialize_ulong_long(s, sf->index);
|
||||
}
|
||||
if (sf->label_point != 0) {
|
||||
serialize_ulong_long(geomfile, sf->label_point, geompos, fname);
|
||||
serialize_ulong_long(s, sf->label_point);
|
||||
}
|
||||
if (sf->extent != 0) {
|
||||
serialize_long_long(geomfile, sf->extent, geompos, fname);
|
||||
serialize_long_long(s, sf->extent);
|
||||
}
|
||||
|
||||
serialize_long_long(geomfile, sf->metapos, geompos, fname);
|
||||
serialize_long_long(s, sf->keys.size());
|
||||
|
||||
if (sf->metapos < 0) {
|
||||
serialize_long_long(geomfile, sf->keys.size(), geompos, fname);
|
||||
|
||||
for (size_t i = 0; i < sf->keys.size(); i++) {
|
||||
serialize_long_long(geomfile, sf->keys[i], geompos, fname);
|
||||
serialize_long_long(geomfile, sf->values[i], geompos, fname);
|
||||
}
|
||||
for (size_t i = 0; i < sf->keys.size(); i++) {
|
||||
serialize_long_long(s, sf->keys[i]);
|
||||
serialize_long_long(s, sf->values[i]);
|
||||
}
|
||||
|
||||
if (include_minzoom) {
|
||||
serialize_byte(geomfile, sf->feature_minzoom, geompos, fname);
|
||||
}
|
||||
// MAGIC: This knows that the feature minzoom is the last byte of the feature,
|
||||
serialize_byte(s, sf->feature_minzoom);
|
||||
return s;
|
||||
}
|
||||
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature deserialize_feature(std::string &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) {
|
||||
serial_feature sf;
|
||||
char *cp = (char *) geoms.c_str();
|
||||
|
||||
deserialize_byte_io(geoms, &sf.t, geompos_in);
|
||||
if (sf.t < 0) {
|
||||
return sf;
|
||||
}
|
||||
|
||||
deserialize_long_long_io(geoms, &sf.layer, geompos_in);
|
||||
deserialize_byte(&cp, &sf.t);
|
||||
deserialize_long_long(&cp, &sf.layer);
|
||||
|
||||
sf.seq = 0;
|
||||
if (sf.layer & (1 << FLAG_SEQ)) {
|
||||
deserialize_long_long_io(geoms, &sf.seq, geompos_in);
|
||||
deserialize_long_long(&cp, &sf.seq);
|
||||
}
|
||||
|
||||
sf.tippecanoe_minzoom = -1;
|
||||
@@ -276,64 +261,54 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_
|
||||
sf.id = 0;
|
||||
sf.has_id = false;
|
||||
if (sf.layer & (1 << FLAG_MINZOOM)) {
|
||||
deserialize_int_io(geoms, &sf.tippecanoe_minzoom, geompos_in);
|
||||
deserialize_int(&cp, &sf.tippecanoe_minzoom);
|
||||
}
|
||||
if (sf.layer & (1 << FLAG_MAXZOOM)) {
|
||||
deserialize_int_io(geoms, &sf.tippecanoe_maxzoom, geompos_in);
|
||||
deserialize_int(&cp, &sf.tippecanoe_maxzoom);
|
||||
}
|
||||
if (sf.layer & (1 << FLAG_ID)) {
|
||||
sf.has_id = true;
|
||||
deserialize_ulong_long_io(geoms, &sf.id, geompos_in);
|
||||
deserialize_ulong_long(&cp, &sf.id);
|
||||
}
|
||||
|
||||
deserialize_int_io(geoms, &sf.segment, geompos_in);
|
||||
deserialize_int(&cp, &sf.segment);
|
||||
|
||||
sf.index = 0;
|
||||
sf.label_point = 0;
|
||||
sf.extent = 0;
|
||||
|
||||
sf.geometry = decode_geometry(geoms, geompos_in, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
|
||||
sf.geometry = decode_geometry(&cp, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]);
|
||||
|
||||
if (sf.layer & (1 << FLAG_INDEX)) {
|
||||
deserialize_ulong_long_io(geoms, &sf.index, geompos_in);
|
||||
deserialize_ulong_long(&cp, &sf.index);
|
||||
}
|
||||
if (sf.layer & (1 << FLAG_LABEL_POINT)) {
|
||||
deserialize_ulong_long_io(geoms, &sf.label_point, geompos_in);
|
||||
deserialize_ulong_long(&cp, &sf.label_point);
|
||||
}
|
||||
if (sf.layer & (1 << FLAG_EXTENT)) {
|
||||
deserialize_long_long_io(geoms, &sf.extent, geompos_in);
|
||||
deserialize_long_long(&cp, &sf.extent);
|
||||
}
|
||||
|
||||
sf.layer >>= FLAG_LAYER;
|
||||
|
||||
sf.metapos = 0;
|
||||
deserialize_long_long_io(geoms, &sf.metapos, geompos_in);
|
||||
long long count;
|
||||
deserialize_long_long(&cp, &count);
|
||||
|
||||
if (sf.metapos >= 0) {
|
||||
char *meta = metabase + sf.metapos + meta_off[sf.segment];
|
||||
long long count;
|
||||
deserialize_long_long(&meta, &count);
|
||||
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long(&meta, &k);
|
||||
deserialize_long_long(&meta, &v);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
} else {
|
||||
long long count;
|
||||
deserialize_long_long_io(geoms, &count, geompos_in);
|
||||
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long_io(geoms, &k, geompos_in);
|
||||
deserialize_long_long_io(geoms, &v, geompos_in);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
for (long long i = 0; i < count; i++) {
|
||||
long long k, v;
|
||||
deserialize_long_long(&cp, &k);
|
||||
deserialize_long_long(&cp, &v);
|
||||
sf.keys.push_back(k);
|
||||
sf.values.push_back(v);
|
||||
}
|
||||
|
||||
deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in);
|
||||
// MAGIC: This knows that the feature minzoom is the last byte of the feature.
|
||||
deserialize_byte(&cp, &sf.feature_minzoom);
|
||||
|
||||
if (cp != geoms.c_str() + geoms.size()) {
|
||||
fprintf(stderr, "wrong length decoding feature: used %zd, len is %zu\n", cp - geoms.c_str(), geoms.size());
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
|
||||
return sf;
|
||||
}
|
||||
@@ -427,6 +402,34 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
sf.bbox[2] = LLONG_MIN;
|
||||
sf.bbox[3] = LLONG_MIN;
|
||||
|
||||
for (size_t i = 0; i < sf.geometry.size(); i++) {
|
||||
if (sf.geometry[i].op == VT_MOVETO || sf.geometry[i].op == VT_LINETO) {
|
||||
if (sf.geometry[i].y > 0 && sf.geometry[i].y < 0xFFFFFFFF) {
|
||||
// standard -180 to 180 world plane
|
||||
|
||||
long long x = sf.geometry[i].x & 0xFFFFFFFF;
|
||||
long long y = sf.geometry[i].y & 0xFFFFFFFF;
|
||||
|
||||
r->file_bbox1[0] = std::min(r->file_bbox1[0], x);
|
||||
r->file_bbox1[1] = std::min(r->file_bbox1[1], y);
|
||||
r->file_bbox1[2] = std::max(r->file_bbox1[2], x);
|
||||
r->file_bbox1[3] = std::max(r->file_bbox1[3], y);
|
||||
|
||||
// printf("%llx,%llx %llx,%llx %llx,%llx ", x, y, r->file_bbox1[0], r->file_bbox1[1], r->file_bbox1[2], r->file_bbox1[3]);
|
||||
|
||||
// shift the western hemisphere 360 degrees to the east
|
||||
if (x < 0x80000000) { // prime meridian
|
||||
x += 0x100000000;
|
||||
}
|
||||
|
||||
r->file_bbox2[0] = std::min(r->file_bbox2[0], x);
|
||||
r->file_bbox2[1] = std::min(r->file_bbox2[1], y);
|
||||
r->file_bbox2[2] = std::max(r->file_bbox2[2], x);
|
||||
r->file_bbox2[3] = std::max(r->file_bbox2[3], y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// try to remind myself that the geometry in this function is in SCALED COORDINATES
|
||||
drawvec scaled_geometry = sf.geometry;
|
||||
sf.geometry.clear();
|
||||
@@ -512,32 +515,6 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
locs.clear();
|
||||
}
|
||||
|
||||
bool inline_meta = true;
|
||||
// Don't inline metadata for features that will span several tiles at maxzoom
|
||||
if (scaled_geometry.size() > 0 && (sf.bbox[2] < sf.bbox[0] || sf.bbox[3] < sf.bbox[1])) {
|
||||
fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", sf.bbox[0], sf.bbox[1], sf.bbox[2], sf.bbox[3]);
|
||||
}
|
||||
if (sf.bbox[0] == LLONG_MAX) {
|
||||
// No bounding box (empty geometry)
|
||||
// Shouldn't happen, but avoid arithmetic overflow below
|
||||
} else if (sf.bbox[2] - sf.bbox[0] > (2LL << (32 - sst->maxzoom)) || sf.bbox[3] - sf.bbox[1] > (2LL << (32 - sst->maxzoom))) {
|
||||
inline_meta = false;
|
||||
|
||||
if (prevent[P_CLIPPING]) {
|
||||
static std::atomic<long long> warned(0);
|
||||
long long extent = ((sf.bbox[2] - sf.bbox[0]) / ((1LL << (32 - sst->maxzoom)) + 1)) * ((sf.bbox[3] - sf.bbox[1]) / ((1LL << (32 - sst->maxzoom)) + 1));
|
||||
if (extent > warned) {
|
||||
fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", sst->fname, sst->line, extent);
|
||||
warned = extent;
|
||||
|
||||
if (extent > 10000) {
|
||||
fprintf(stderr, "Exiting because this can't be right.\n");
|
||||
exit(EXIT_IMPOSSIBLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
double extent = 0;
|
||||
if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED] || order_by_size || sst->want_dist) {
|
||||
if (sf.t == VT_POLYGON) {
|
||||
@@ -725,24 +702,17 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
}
|
||||
}
|
||||
|
||||
if (inline_meta) {
|
||||
sf.metapos = -1;
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
|
||||
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
|
||||
}
|
||||
} else {
|
||||
sf.metapos = r->metapos;
|
||||
serialize_long_long(r->metafile, sf.full_keys.size(), &r->metapos, sst->fname);
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string), &r->metapos, sst->fname);
|
||||
serialize_long_long(r->metafile, addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type), &r->metapos, sst->fname);
|
||||
}
|
||||
for (size_t i = 0; i < sf.full_keys.size(); i++) {
|
||||
sf.keys.push_back(addpool(r->poolfile, r->treefile, sf.full_keys[i].c_str(), mvt_string));
|
||||
sf.values.push_back(addpool(r->poolfile, r->treefile, sf.full_values[i].s.c_str(), sf.full_values[i].type));
|
||||
}
|
||||
|
||||
long long geomstart = r->geompos;
|
||||
sf.geometry = scaled_geometry;
|
||||
serialize_feature(r->geomfile, &sf, &r->geompos, sst->fname, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y)), false);
|
||||
|
||||
std::string feature = serialize_feature(&sf, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y)));
|
||||
serialize_long_long(r->geomfile, feature.size(), &r->geompos, sst->fname);
|
||||
fwrite_check(feature.c_str(), sizeof(char), feature.size(), r->geomfile, &r->geompos, sst->fname);
|
||||
|
||||
struct index index;
|
||||
index.start = geomstart;
|
||||
@@ -752,8 +722,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
index.t = sf.t;
|
||||
index.ix = bbox_index;
|
||||
|
||||
fwrite_check(&index, sizeof(struct index), 1, r->indexfile, sst->fname);
|
||||
r->indexpos += sizeof(struct index);
|
||||
fwrite_check(&index, sizeof(struct index), 1, r->indexfile, &r->indexpos, sst->fname);
|
||||
|
||||
for (size_t i = 0; i < 2; i++) {
|
||||
if (sf.bbox[i] < r->file_bbox[i]) {
|
||||
@@ -770,6 +739,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) {
|
||||
checkdisk(sst->readers);
|
||||
if (!quiet && !quiet_progress && progress_time()) {
|
||||
fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0);
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
(*(sst->progress_seq))++;
|
||||
|
||||
+14
-24
@@ -11,14 +11,19 @@
|
||||
#include "mbtiles.hpp"
|
||||
#include "jsonpull/jsonpull.h"
|
||||
|
||||
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname);
|
||||
size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, std::atomic<long long> *fpos, const char *fname);
|
||||
|
||||
void serialize_int(FILE *out, int n, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_long_long(FILE *out, long long n, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_ulong_long(FILE *out, unsigned long long n, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_byte(FILE *out, signed char n, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_uint(FILE *out, unsigned n, std::atomic<long long> *fpos, const char *fname);
|
||||
void serialize_string(FILE *out, const char *s, std::atomic<long long> *fpos, const char *fname);
|
||||
|
||||
void serialize_int(std::string &out, int n);
|
||||
void serialize_long_long(std::string &out, long long n);
|
||||
void serialize_ulong_long(std::string &out, unsigned long long n);
|
||||
void serialize_byte(std::string &out, signed char n);
|
||||
void serialize_uint(std::string &out, unsigned n);
|
||||
|
||||
void deserialize_int(char **f, int *n);
|
||||
void deserialize_long_long(char **f, long long *n);
|
||||
@@ -26,12 +31,6 @@ void deserialize_ulong_long(char **f, unsigned long long *n);
|
||||
void deserialize_uint(char **f, unsigned *n);
|
||||
void deserialize_byte(char **f, signed char *n);
|
||||
|
||||
int deserialize_int_io(FILE *f, int *n, std::atomic<long long> *geompos);
|
||||
int deserialize_long_long_io(FILE *f, long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_ulong_long_io(FILE *f, unsigned long long *n, std::atomic<long long> *geompos);
|
||||
int deserialize_uint_io(FILE *f, unsigned *n, std::atomic<long long> *geompos);
|
||||
int deserialize_byte_io(FILE *f, signed char *n, std::atomic<long long> *geompos);
|
||||
|
||||
struct serial_val {
|
||||
int type = 0;
|
||||
std::string s = "";
|
||||
@@ -61,8 +60,6 @@ struct serial_feature {
|
||||
|
||||
std::vector<long long> keys{};
|
||||
std::vector<long long> values{};
|
||||
// If >= 0, metadata is external
|
||||
long long metapos = 0;
|
||||
|
||||
// XXX This isn't serialized. Should it be here?
|
||||
long long bbox[4] = {0, 0, 0, 0};
|
||||
@@ -72,54 +69,48 @@ struct serial_feature {
|
||||
bool dropped = false;
|
||||
};
|
||||
|
||||
void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic<long long> *geompos, const char *fname, long long wx, long long wy, bool include_minzoom);
|
||||
serial_feature deserialize_feature(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, long long *meta_off, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
std::string serialize_feature(serial_feature *sf, long long wx, long long wy);
|
||||
serial_feature deserialize_feature(std::string &geoms, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y);
|
||||
|
||||
struct reader {
|
||||
int metafd = -1;
|
||||
int poolfd = -1;
|
||||
int treefd = -1;
|
||||
int geomfd = -1;
|
||||
int indexfd = -1;
|
||||
|
||||
FILE *metafile = NULL;
|
||||
struct memfile *poolfile = NULL;
|
||||
struct memfile *treefile = NULL;
|
||||
FILE *geomfile = NULL;
|
||||
FILE *indexfile = NULL;
|
||||
|
||||
std::atomic<long long> metapos;
|
||||
std::atomic<long long> geompos;
|
||||
std::atomic<long long> indexpos;
|
||||
|
||||
long long file_bbox[4] = {0, 0, 0, 0};
|
||||
|
||||
long long file_bbox1[4] = {0xFFFFFFFF, 0xFFFFFFFF, 0, 0}; // standard -180 to 180 world plane
|
||||
long long file_bbox2[4] = {0x1FFFFFFFF, 0xFFFFFFFF, 0x100000000, 0}; // 0 to 360 world plane
|
||||
|
||||
struct stat geomst {};
|
||||
struct stat metast {};
|
||||
|
||||
char *geom_map = NULL;
|
||||
|
||||
reader()
|
||||
: metapos(0), geompos(0), indexpos(0) {
|
||||
: geompos(0), indexpos(0) {
|
||||
}
|
||||
|
||||
reader(reader const &r) {
|
||||
metafd = r.metafd;
|
||||
poolfd = r.poolfd;
|
||||
treefd = r.treefd;
|
||||
geomfd = r.geomfd;
|
||||
indexfd = r.indexfd;
|
||||
|
||||
metafile = r.metafile;
|
||||
poolfile = r.poolfile;
|
||||
treefile = r.treefile;
|
||||
geomfile = r.geomfile;
|
||||
indexfile = r.indexfile;
|
||||
|
||||
long long p = r.metapos;
|
||||
metapos = p;
|
||||
|
||||
p = r.geompos;
|
||||
long long p = r.geompos;
|
||||
geompos = p;
|
||||
|
||||
p = r.indexpos;
|
||||
@@ -128,7 +119,6 @@ struct reader {
|
||||
memcpy(file_bbox, r.file_bbox, sizeof(file_bbox));
|
||||
|
||||
geomst = r.geomst;
|
||||
metast = r.metast;
|
||||
|
||||
geom_map = r.geom_map;
|
||||
}
|
||||
|
||||
+65
-64
File diff suppressed because one or more lines are too long
+1
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "3.757458,-84.969764,359.744029,85.007238",
|
||||
"bounds": "-179.320808,-85.051129,177.449262,85.051129",
|
||||
"center": "-174.375000,-52.349536,5",
|
||||
"description": "tests/accumulate/out/-z5_-Ethesum@sum_-Etheproduct@product_-Ethemax@max_-Ethemin@min_-Ethemean@mean_-Etheconcat@concat_-Ethecomma@comma.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.373782,37.454389,-121.469214,37.905824",
|
||||
"bounds": "-122.373782,37.454389,-121.469214,37.905824",
|
||||
"center": "-121.992188,37.454389,9",
|
||||
"description": "tests/allow-existing/both.mbtiles",
|
||||
|
||||
+1
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/attribute-type/out/-z0_-Tinttype@int_-Tfloattype@float_-Tbooltype@bool_-Tstringtype@string.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/attribute-type/out/-z0_-pN.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "13.365261,39.637013,23.009582,46.863962",
|
||||
"bounds": "13.365261,39.637013,23.009582,46.863962",
|
||||
"center": "23.009582,42.525564,1",
|
||||
"description": "tests/border/out/-z1_--detect-shared-borders.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.023803,-85.040752,359.950215,83.645130",
|
||||
"bounds": "-180.000000,-85.051129,180.000000,83.645130",
|
||||
"center": "90.000000,42.525564,1",
|
||||
"description": "tests/coalesce-id/out/-z1_--coalesce_--reorder.json.check.mbtiles",
|
||||
|
||||
+1
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-123.173825,37.454389,-121.469214,37.929824",
|
||||
"bounds": "-123.173825,37.454389,-121.469214,37.929824",
|
||||
"center": "-122.255859,37.788049,11",
|
||||
"description": "tests/coalesce-tract/out/-P_--coalesce_--reorder_-z11_-Z11_-y_STATEFP10_-y_COUNTYFP10_-l_merged.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-175.220565,-41.299988,179.216647,64.150024",
|
||||
"bounds": "-180.000000,-41.299988,180.000000,85.051129",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/csv/out-null.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-175.220565,-41.299988,179.216647,64.150024",
|
||||
"bounds": "-180.000000,-41.299988,180.000000,85.051129",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/csv/out.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "90.000000,-82.000000,332.000000,83.000000",
|
||||
"bounds": "-139.000000,-82.000000,170.000000,83.000000",
|
||||
"center": "-135.000000,75.782195,2",
|
||||
"description": "tests/curve/out/-z2.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "90.000000,-82.000000,332.000000,83.000000",
|
||||
"bounds": "-139.000000,-82.000000,170.000000,83.000000",
|
||||
"center": "-135.000000,75.782195,2",
|
||||
"description": "tests/curve/out/-z2_--no-clipping.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "90.000000,-82.000000,332.000000,83.000000",
|
||||
"bounds": "-139.000000,-82.000000,170.000000,83.000000",
|
||||
"center": "45.000000,33.256630,2",
|
||||
"description": "tests/curve/out/-z2_--no-duplication.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-180.000000,0.000000,177.539063,68.269386",
|
||||
"bounds": "-180.000000,0.000000,180.000000,68.269386",
|
||||
"center": "-174.375000,16.560724,5",
|
||||
"description": "tests/dateline/out/-z5.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-180.000000,0.000000,177.539063,68.269386",
|
||||
"bounds": "-180.000000,0.000000,180.000000,68.269386",
|
||||
"center": "174.375000,44.951199,5",
|
||||
"description": "tests/dateline/out/-z5_-b0.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.822234,37.723073,-122.249782,38.090533",
|
||||
"bounds": "-122.822234,37.723073,-122.249782,38.090533",
|
||||
"center": "-122.249782,37.796763,14",
|
||||
"description": "tests/empty-linestring/out/-ac.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-175.220565,-41.299974,179.216647,64.150024",
|
||||
"bounds": "-175.220565,-41.299974,179.216647,64.150024",
|
||||
"center": "16.875000,44.951199,5",
|
||||
"description": "tests/epsg-3857/out/-yNAME_-z5_-sEPSG@3857.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-101.000000,0.000000,1.000000,1.000000",
|
||||
"bounds": "-101.000000,0.000000,1.000000,1.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/feature-filter/out/-z0_-Jtests%feature-filter%filter.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129",
|
||||
"bounds": "-180.000000,-85.051129,180.000000,85.051129",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/feature-filter/out/all.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-175.781250,-42.032974,180.000000,64.168107",
|
||||
"bounds": "-175.781250,-42.032974,180.000000,64.168107",
|
||||
"center": "-62.578125,17.307462,8",
|
||||
"description": "tests/feature-filter/out/places.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.000000,-23.563987,102.000000,59.900000",
|
||||
"bounds": "-122.000000,-23.563987,102.000000,59.900000",
|
||||
"center": "22.500000,20.489949,3",
|
||||
"description": "tests/geometry/out/-z3.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.021973,-0.065918,0.065918,-0.021973",
|
||||
"bounds": "0.021973,-0.065918,0.065918,-0.021973",
|
||||
"center": "0.065918,-0.065918,11",
|
||||
"description": "tests/grid-aligned/out/-z11_-D7_--grid-low-zooms.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.084070,0.057600",
|
||||
"bounds": "0.000000,0.000000,0.084070,0.057600",
|
||||
"center": "0.084070,0.057600,11",
|
||||
"description": "tests/grid-unaligned/out/-z11_-D7_--grid-low-zooms.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-156.000000,30.000000,-106.000000,35.000000",
|
||||
"bounds": "-180.000000,30.000000,180.000000,35.000000",
|
||||
"center": "-90.000000,35.000000,1",
|
||||
"description": "tests/high-longitude/out/-z1.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.000000,37.000000,0.000000,51.500000",
|
||||
"bounds": "-122.000000,37.000000,0.000000,51.500000",
|
||||
"center": "-122.000000,37.000008,24",
|
||||
"description": "tests/highzoom/out/-z30.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.294563,37.695450,-122.104420,37.833010",
|
||||
"bounds": "-122.294563,37.695450,-122.104420,37.833010",
|
||||
"center": "-122.156982,37.762029,14",
|
||||
"description": "tests/id/out/-Z11.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "138.063812,-10.009860,209.793590,9.775580",
|
||||
"bounds": "-174.543405,-10.009860,174.245778,9.775580",
|
||||
"center": "139.218750,9.775580,7",
|
||||
"description": "tests/islands/out/-d7_-z7_-pt_-pp.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129",
|
||||
"bounds": "-180.000000,-85.051129,180.000000,85.051129",
|
||||
"center": "-122.104097,37.695438,0",
|
||||
"description": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "180.000000,85.051129,-180.000000,-85.051129",
|
||||
"bounds": "-180.000000,-85.051129,180.000000,-85.051129",
|
||||
"center": "0.000000,-85.051129,0",
|
||||
"format": "pbf",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
|
||||
"attribution": "macarthur's attribution",
|
||||
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"center": "-122.167969,37.833010,9",
|
||||
"description": "tests/join-population/macarthur.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"center": "-122.167969,37.833010,9",
|
||||
"description": "tests/join-population/macarthur.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
|
||||
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420-folder",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
|
||||
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
|
||||
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
|
||||
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
|
||||
"center": "-122.299805,37.892187,12",
|
||||
"description": "tests/join-population/tabblock_06001420.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"center": "-122.167969,37.828608,10",
|
||||
"description": "tests/join-population/macarthur2.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
|
||||
"center": "-122.167969,37.833010,10",
|
||||
"description": "tests/join-population/macarthur.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-87.564090,38.490831,-87.251603,38.902199",
|
||||
"bounds": "-87.564090,38.490831,-87.251603,38.902199",
|
||||
"center": "-87.363281,38.685378,10",
|
||||
"description": "tests/knox/out/-zg.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-87.564090,38.490831,-87.251603,38.902199",
|
||||
"bounds": "-87.564090,38.490831,-87.251603,38.902199",
|
||||
"center": "-87.363281,38.685378,10",
|
||||
"description": "tests/knox/out/-zg_-P.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-175.220564,-41.299973,179.216647,64.150023",
|
||||
"bounds": "-175.220564,-41.299973,179.216647,64.150023",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/layer-json/out.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/longattr/out/-z0.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/longjson/out/-z0.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"bounds": "0.000000,0.000000,0.000000,0.000000",
|
||||
"center": "0.000000,0.000000,0",
|
||||
"description": "tests/longlayer/out/-z0.json.check.mbtiles",
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"center": "1.000000,1.000000,0",
|
||||
"description": "tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles",
|
||||
"format": "pbf",
|
||||
"generator_options": "./tippecanoe -q -a@ -f -o tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles -z0 -O200 --cluster-densest-as-needed tests/loop/in.json",
|
||||
"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{\"clustered\":\"Boolean\",\"point_count\":\"Number\",\"sqrt_point_count\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":1000,\"geometry\":\"Point\",\"attributeCount\":3,\"attributes\":[{\"attribute\":\"clustered\",\"count\":1,\"type\":\"boolean\",\"values\":[true]},{\"attribute\":\"point_count\",\"count\":1,\"type\":\"number\",\"values\":[1000],\"min\":1000,\"max\":1000},{\"attribute\":\"sqrt_point_count\",\"count\":1,\"type\":\"number\",\"values\":[31.620000],\"min\":31.62,\"max\":31.62}]}]}}",
|
||||
"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":0,\"fields\":{\"clustered\":\"Boolean\",\"point_count\":\"Number\",\"point_count_abbreviated\":\"String\",\"sqrt_point_count\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":1000,\"geometry\":\"Point\",\"attributeCount\":4,\"attributes\":[{\"attribute\":\"clustered\",\"count\":1,\"type\":\"boolean\",\"values\":[true]},{\"attribute\":\"point_count\",\"count\":1,\"type\":\"number\",\"values\":[1000],\"min\":1000,\"max\":1000},{\"attribute\":\"point_count_abbreviated\",\"count\":1,\"type\":\"string\",\"values\":[\"1.0k\"]},{\"attribute\":\"sqrt_point_count\",\"count\":1,\"type\":\"number\",\"values\":[31.620000],\"min\":31.62,\"max\":31.62}]}]}}",
|
||||
"maxzoom": "0",
|
||||
"minzoom": "0",
|
||||
"name": "tests/loop/out/-z0_-O200_--cluster-densest-as-needed.json.check.mbtiles",
|
||||
@@ -14,7 +15,7 @@
|
||||
}, "features": [
|
||||
{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
|
||||
{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
|
||||
{ "type": "Feature", "properties": { "clustered": true, "point_count": 1000, "sqrt_point_count": 31.62 }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
|
||||
{ "type": "Feature", "properties": { "clustered": true, "point_count": 1000, "sqrt_point_count": 31.62, "point_count_abbreviated": "1.0k" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 0.966751 ] } }
|
||||
] }
|
||||
] }
|
||||
] }
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"center": "1.000000,1.000000,0",
|
||||
"description": "tests/loop/out/-z0_-O200_--drop-densest-as-needed.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"bounds": "1.000000,1.000000,1.000000,1.000000",
|
||||
"center": "1.000000,1.000000,0",
|
||||
"description": "tests/loop/out/-z0_-O200_--drop-fraction-as-needed.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "0.000000,0.000000,6.000000,6.000000",
|
||||
"bounds": "0.000000,0.000000,6.000000,6.000000",
|
||||
"center": "6.000000,6.000000,6",
|
||||
"description": "tests/minzoom/out/-z6.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-124.213807,25.789556,-70.645734,49.005639",
|
||||
"bounds": "-124.213807,25.789556,-70.645734,49.005639",
|
||||
"center": "-70.645734,25.789556,3",
|
||||
"description": "tests/multilayer/out/-ltogether_-z3.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-124.213807,25.789556,-70.645734,49.005639",
|
||||
"bounds": "-124.213807,25.789556,-70.645734,49.005639",
|
||||
"center": "-70.645734,25.789556,3",
|
||||
"description": "separate",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-34.101500,-16.299000,33.046000,70.728000",
|
||||
"bounds": "-34.101500,-16.299000,33.046000,70.728000",
|
||||
"center": "-34.101500,42.525564,1",
|
||||
"description": "tests/multilinestring/out/-z1.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/decode/multi.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/decode/multi.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/decode/multi.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/out/-Z11_-z11.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/out/-Z11_-z11_--calculate-feature-density.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/out/-Z11_-z11_--hilbert.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/out/-Z11_-z11_--prefer-radix-sort.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.431641,37.788049,11",
|
||||
"description": "tests/muni/out/-Z11_-z11_-g2.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-B15.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-Bf2000.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-M10000.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-M10000_-aG.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-M10000_-ad.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-M10000_-pd.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.497559,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-M5000_-as.json.check.mbtiles",
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-rf2000.json.check.mbtiles",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ "type": "FeatureCollection", "properties": {
|
||||
"antimeridian_adjusted_bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"bounds": "-122.538670,37.705764,-12.240000,37.836443",
|
||||
"center": "-122.409668,37.770713,13",
|
||||
"description": "tests/muni/out/-Z11_-z13_-rf2000_-Bg.json.check.mbtiles",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user