diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into tile-join-sqlite

Browse Source
This commit is contained in:
Erica Fischer
2024-12-17 15:23:01 -08:00
parent fcc611adc4 7165ae6999
commit 638a7617d0
11 changed files with 136 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test.yml
+8 -2
View File
@@ -4,12 +4,18 @@ on: [push]
jobs:
test:
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
version: ['Release', 'Debug']
steps:
- uses: actions/checkout@v3
- run: sudo apt-get install libsqlite3-dev
- name: Install dependencies (Ubuntu)
if: runner.os == 'Linux'
run: sudo apt-get install libsqlite3-dev
- name: Install dependencies (macOS)
if: runner.os == 'macOS'
run: brew install sqlite3
- run: uname -a; BUILDTYPE=${{ matrix.version }} make
- run: make test
CHANGELOG.md
+4
View File
@@ -1,3 +1,7 @@
# 2.73.0
* Correctly clip features down to nothing when the clip region doesn't intersect the tile at all
# 2.72.0
* Add --clip-polygon-file and --feature-filter-file options to tippecanoe-overzoom
Makefile
+6 -2
View File
@@ -59,7 +59,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I. -Iclipper2/include
LIBS = -L/usr/local/lib
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-enumerate: enumerate.o
@@ -68,7 +68,7 @@ tippecanoe-enumerate: enumerate.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
@@ -439,6 +439,10 @@ overzoom-test: tippecanoe-overzoom
./tippecanoe-decode tests/pbf/countries-8-135-86-bigclip.pbf 8 135 86 > tests/pbf/countries-8-135-86-bigclip.json.check
cmp tests/pbf/countries-8-135-86-bigclip.json.check tests/pbf/countries-8-135-86-bigclip.json
rm tests/pbf/countries-8-135-86-bigclip.pbf tests/pbf/countries-8-135-86-bigclip.json.check
# Clip region that does not intersect with the tile
./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077
cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null # clipped away
rm tests/pbf/squirrels-13-2413-3077-clip.pbf
join-test: tippecanoe tippecanoe-decode tile-join
./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'
main.cpp
+9 -40
View File
@@ -21,7 +21,6 @@
#include <limits.h>
#include <sqlite3.h>
#include <stdarg.h>
#include <sys/resource.h>
#include <pthread.h>
#include <getopt.h>
#include <signal.h>
@@ -67,6 +66,7 @@
#include "sort.hpp"
#include "attribute.hpp"
#include "thread.hpp"
#include "platform.hpp"
static int low_detail = 12;
static int full_detail = -1;
@@ -187,7 +187,7 @@ void init_cpus() {
if (TIPPECANOE_MAX_THREADS != NULL) {
CPUS = atoi_require(TIPPECANOE_MAX_THREADS, "TIPPECANOE_MAX_THREADS");
} else {
CPUS = sysconf(_SC_NPROCESSORS_ONLN);
CPUS = get_num_avail_cpus();
}
if (CPUS < 1) {
@@ -202,13 +202,7 @@ void init_cpus() {
// Round down to a power of 2
CPUS = 1 << (int) (log(CPUS) / log(2));
struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit");
exit(EXIT_PTHREAD);
} else {
MAX_FILES = rl.rlim_cur;
}
MAX_FILES = get_max_open_files();
// Don't really want too many temporary files, because the file system
// will start to bog down eventually
@@ -222,7 +216,7 @@ void init_cpus() {
long long fds[MAX_FILES];
long long i;
for (i = 0; i < MAX_FILES; i++) {
fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC);
fds[i] = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (fds[i] < 0) {
break;
}
@@ -899,7 +893,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
std::atomic<long long> indexpos(indexst.st_size);
int bytes = sizeof(struct index);
int page = sysconf(_SC_PAGESIZE);
int page = get_page_size();
// Don't try to sort more than 2GB at once,
// which used to crash Macs and may still
long long max_unit = 2LL * 1024 * 1024 * 1024;
@@ -1069,31 +1063,6 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
}
}
static size_t calc_memsize() {
size_t mem;
#ifdef __APPLE__
int64_t hw_memsize;
size_t len = sizeof(int64_t);
if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
perror("sysctl hw.memsize");
exit(EXIT_MEMORY);
}
mem = hw_memsize;
#else
long long pagesize = sysconf(_SC_PAGESIZE);
long long pages = sysconf(_SC_PHYS_PAGES);
if (pages < 0 || pagesize < 0) {
perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
exit(EXIT_MEMORY);
}
mem = (long long) pages * pagesize;
#endif
return mem;
}
void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
// Run through the index and geometry for each reader,
// splitting the contents out by index into as many
@@ -1446,7 +1415,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
size_t dist_count = 0;
double area_sum = 0;
int files_open_before_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
int files_open_before_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_before_reading < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
@@ -1887,7 +1856,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
}
}
int files_open_after_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
int files_open_after_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_after_reading < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
@@ -3728,7 +3697,7 @@ int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC);
files_open_at_start = open(get_null_device(), O_RDONLY | O_CLOEXEC);
if (files_open_at_start < 0) {
perror("open /dev/null");
exit(EXIT_OPEN);
@@ -3878,7 +3847,7 @@ int main(int argc, char **argv) {
muntrace();
#endif
i = open("/dev/null", O_RDONLY | O_CLOEXEC);
i = open(get_null_device(), O_RDONLY | O_CLOEXEC);
// i < files_open_at_start is not an error, because reading from a pipe closes stdin
if (i > files_open_at_start) {
fprintf(stderr, "Internal error: did not close all files: %d\n", i);
overzoom.cpp
+31 -21
View File
@@ -294,6 +294,7 @@ int main(int argc, char **argv) {
// clip the clip polygons, if any, to the tile bounds,
// to reduce their complexity
bool clipped_to_nothing = false;
if (clipbboxes.size() > 0) {
long long wx1 = (nx - buffer / 256.0) * (1LL << (32 - nz));
long long wy1 = (ny - buffer / 256.0) * (1LL << (32 - nz));
@@ -315,38 +316,47 @@ int main(int argc, char **argv) {
if (c.dv.size() > 0) {
c.dv = clip_poly_poly(c.dv, tile_bounds);
if (c.dv.size() == 0) {
clipped_to_nothing = true;
break;
}
}
}
}
json_object *json_filter = NULL;
if (filter.size() > 0) {
json_filter = parse_filter(filter.c_str());
}
std::string out;
for (auto const &s : sources) {
std::string tile;
char buf[1000];
int len;
FILE *f = fopen(s.tile.c_str(), "rb");
if (f == NULL) {
perror(s.tile.c_str());
exit(EXIT_FAILURE);
if (!clipped_to_nothing) {
json_object *json_filter = NULL;
if (filter.size() > 0) {
json_filter = parse_filter(filter.c_str());
}
while ((len = fread(buf, sizeof(char), 1000, f)) > 0) {
tile.append(std::string(buf, len));
for (auto const &s : sources) {
std::string tile;
char buf[1000];
int len;
FILE *f = fopen(s.tile.c_str(), "rb");
if (f == NULL) {
perror(s.tile.c_str());
exit(EXIT_FAILURE);
}
while ((len = fread(buf, sizeof(char), 1000, f)) > 0) {
tile.append(std::string(buf, len));
}
fclose(f);
input_tile t = s;
t.tile = std::move(tile);
its.push_back(std::move(t));
}
fclose(f);
input_tile t = s;
t.tile = std::move(tile);
its.push_back(std::move(t));
out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
}
std::string out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
FILE *f = fopen(outfile, "wb");
if (f == NULL) {
perror(outfile);
platform.cpp
+56
View File
@@ -0,0 +1,56 @@
#include "platform.hpp"
#include <cstdlib>
#include <cstdio>
#include <unistd.h>
#include <sys/resource.h>
#if defined(__APPLE__) || defined(__FreeBSD__)
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/param.h>
#include <sys/mount.h>
#endif
#include "errors.hpp"
long get_num_avail_cpus() {
return sysconf(_SC_NPROCESSORS_ONLN);
}
long get_page_size() {
return sysconf(_SC_PAGESIZE);
}
size_t calc_memsize() {
size_t mem;
#ifdef __APPLE__
int64_t hw_memsize;
size_t len = sizeof(int64_t);
if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
perror("sysctl hw.memsize");
exit(EXIT_MEMORY);
}
mem = hw_memsize;
#else
long long pagesize = sysconf(_SC_PAGESIZE);
long long pages = sysconf(_SC_PHYS_PAGES);
if (pages < 0 || pagesize < 0) {
perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
exit(EXIT_MEMORY);
}
mem = (long long) pages * pagesize;
#endif
return mem;
}
size_t get_max_open_files() {
struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
perror("getrlimit");
exit(EXIT_PTHREAD);
}
return rl.rlim_cur;
}
platform.hpp
+18
View File
@@ -0,0 +1,18 @@
#ifndef PLATFORM_HPP
#define PLATFORM_HPP
#include <cstddef>
long get_num_avail_cpus();
long get_page_size();
size_t calc_memsize();
size_t get_max_open_files();
constexpr const char *get_null_device() {
return "/dev/null";
}
#endif // PLATFORM_HPP
tests/pbf/squirrels-13-2413-3077.pbf
BIN
View File
Binary file not shown.
tests/pbf/squirrels-clip.json
+1
View File
@@ -0,0 +1 @@
{"coordinates":[[[-73.9722549,40.7904659],[-73.9755817,40.7674848],[-73.938783,40.767022],[-73.9457082,40.7791562],[-73.9628855,40.7760201],[-73.9629534,40.7914425],[-73.9722549,40.7904659]]],"crs":{"properties":{"name":"EPSG:4326"},"type":"name"},"type":"Polygon"}
tile-join.cpp
+2 -1
View File
@@ -43,6 +43,7 @@
#include "errors.hpp"
#include "geometry.hpp"
#include "thread.hpp"
#include "platform.hpp"
int pk = false;
int pC = false;
@@ -1325,7 +1326,7 @@ int main(int argc, char **argv) {
struct tileset_reader *readers = NULL;
CPUS = sysconf(_SC_NPROCESSORS_ONLN);
CPUS = get_num_avail_cpus();
const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS");
if (TIPPECANOE_MAX_THREADS != NULL) {
version.hpp
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef VERSION_HPP
#define VERSION_HPP
#define VERSION "v2.72.0"
#define VERSION "v2.73.0"
#endif