Merge remote-tracking branch 'origin/main' into tile-join-sqlite

.github/workflows/test.yml

@@ -4,12 +4,18 @@ on: [push]
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-latest, macos-latest]
         version: ['Release', 'Debug']
     steps:
       - uses: actions/checkout@v3
-      - run: sudo apt-get install libsqlite3-dev
+      - name: Install dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: sudo apt-get install libsqlite3-dev
+      - name: Install dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: brew install sqlite3
       - run: uname -a; BUILDTYPE=${{ matrix.version }} make
       - run: make test

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.73.0
+
+* Correctly clip features down to nothing when the clip region doesn't intersect the tile at all
+
 # 2.72.0
 
 * Add --clip-polygon-file and --feature-filter-file options to tippecanoe-overzoom

Makefile

@@ -59,7 +59,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
 INCLUDES = -I/usr/local/include -I. -Iclipper2/include
 LIBS = -L/usr/local/lib
 
-tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
+tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o platform.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o shared_borders.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-enumerate: enumerate.o
@@ -68,7 +68,7 @@ tippecanoe-enumerate: enumerate.o
 tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
 
-tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
+tile-join: tile-join.o platform.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o read_json.o clipper2/src/clipper.engine.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
@@ -439,6 +439,10 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-decode tests/pbf/countries-8-135-86-bigclip.pbf 8 135 86 > tests/pbf/countries-8-135-86-bigclip.json.check
 	cmp tests/pbf/countries-8-135-86-bigclip.json.check tests/pbf/countries-8-135-86-bigclip.json
 	rm tests/pbf/countries-8-135-86-bigclip.pbf tests/pbf/countries-8-135-86-bigclip.json.check
+	# Clip region that does not intersect with the tile
+	./tippecanoe-overzoom -o tests/pbf/squirrels-13-2413-3077-clip.pbf --clip-polygon-file tests/pbf/squirrels-clip.json tests/pbf/squirrels-13-2413-3077.pbf 13/2413/3077 13/2413/3077
+	cmp tests/pbf/squirrels-13-2413-3077-clip.pbf /dev/null  # clipped away
+	rm tests/pbf/squirrels-13-2413-3077-clip.pbf
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'

main.cpp

@@ -21,7 +21,6 @@
 #include <limits.h>
 #include <sqlite3.h>
 #include <stdarg.h>
-#include <sys/resource.h>
 #include <pthread.h>
 #include <getopt.h>
 #include <signal.h>
@@ -67,6 +66,7 @@
 #include "sort.hpp"
 #include "attribute.hpp"
 #include "thread.hpp"
+#include "platform.hpp"
 
 static int low_detail = 12;
 static int full_detail = -1;
@@ -187,7 +187,7 @@ void init_cpus() {
 	if (TIPPECANOE_MAX_THREADS != NULL) {
 		CPUS = atoi_require(TIPPECANOE_MAX_THREADS, "TIPPECANOE_MAX_THREADS");
 	} else {
-		CPUS = sysconf(_SC_NPROCESSORS_ONLN);
+		CPUS = get_num_avail_cpus();
 	}
 
 	if (CPUS < 1) {
@@ -202,13 +202,7 @@ void init_cpus() {
 	// Round down to a power of 2
 	CPUS = 1 << (int) (log(CPUS) / log(2));
 
-	struct rlimit rl;
+	MAX_FILES = get_max_open_files();
-	if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
-		perror("getrlimit");
-		exit(EXIT_PTHREAD);
-	} else {
-		MAX_FILES = rl.rlim_cur;
-	}
 
 	// Don't really want too many temporary files, because the file system
 	// will start to bog down eventually
@@ -222,7 +216,7 @@ void init_cpus() {
 	long long fds[MAX_FILES];
 	long long i;
 	for (i = 0; i < MAX_FILES; i++) {
-		fds[i] = open("/dev/null", O_RDONLY | O_CLOEXEC);
+		fds[i] = open(get_null_device(), O_RDONLY | O_CLOEXEC);
 		if (fds[i] < 0) {
 			break;
 		}
@@ -899,7 +893,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
 				std::atomic<long long> indexpos(indexst.st_size);
 				int bytes = sizeof(struct index);
 
-				int page = sysconf(_SC_PAGESIZE);
+				int page = get_page_size();
 				// Don't try to sort more than 2GB at once,
 				// which used to crash Macs and may still
 				long long max_unit = 2LL * 1024 * 1024 * 1024;
@@ -1069,31 +1063,6 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
 	}
 }
 
-static size_t calc_memsize() {
-	size_t mem;
-
-#ifdef __APPLE__
-	int64_t hw_memsize;
-	size_t len = sizeof(int64_t);
-	if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
-		perror("sysctl hw.memsize");
-		exit(EXIT_MEMORY);
-	}
-	mem = hw_memsize;
-#else
-	long long pagesize = sysconf(_SC_PAGESIZE);
-	long long pages = sysconf(_SC_PHYS_PAGES);
-	if (pages < 0 || pagesize < 0) {
-		perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
-		exit(EXIT_MEMORY);
-	}
-
-	mem = (long long) pages * pagesize;
-#endif
-
-	return mem;
-}
-
 void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
 	// Run through the index and geometry for each reader,
 	// splitting the contents out by index into as many
@@ -1446,7 +1415,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 	size_t dist_count = 0;
 	double area_sum = 0;
 
-	int files_open_before_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
+	int files_open_before_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
 	if (files_open_before_reading < 0) {
 		perror("open /dev/null");
 		exit(EXIT_OPEN);
@@ -1887,7 +1856,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 		}
 	}
 
-	int files_open_after_reading = open("/dev/null", O_RDONLY | O_CLOEXEC);
+	int files_open_after_reading = open(get_null_device(), O_RDONLY | O_CLOEXEC);
 	if (files_open_after_reading < 0) {
 		perror("open /dev/null");
 		exit(EXIT_OPEN);
@@ -3728,7 +3697,7 @@ int main(int argc, char **argv) {
 
 	signal(SIGPIPE, SIG_IGN);
 
-	files_open_at_start = open("/dev/null", O_RDONLY | O_CLOEXEC);
+	files_open_at_start = open(get_null_device(), O_RDONLY | O_CLOEXEC);
 	if (files_open_at_start < 0) {
 		perror("open /dev/null");
 		exit(EXIT_OPEN);
@@ -3878,7 +3847,7 @@ int main(int argc, char **argv) {
 	muntrace();
 #endif
 
-	i = open("/dev/null", O_RDONLY | O_CLOEXEC);
+	i = open(get_null_device(), O_RDONLY | O_CLOEXEC);
 	// i < files_open_at_start is not an error, because reading from a pipe closes stdin
 	if (i > files_open_at_start) {
 		fprintf(stderr, "Internal error: did not close all files: %d\n", i);

overzoom.cpp

@@ -294,6 +294,7 @@ int main(int argc, char **argv) {
 	// clip the clip polygons, if any, to the tile bounds,
 	// to reduce their complexity
 
+	bool clipped_to_nothing = false;
 	if (clipbboxes.size() > 0) {
 		long long wx1 = (nx - buffer / 256.0) * (1LL << (32 - nz));
 		long long wy1 = (ny - buffer / 256.0) * (1LL << (32 - nz));
@@ -315,38 +316,47 @@ int main(int argc, char **argv) {
 
 			if (c.dv.size() > 0) {
 				c.dv = clip_poly_poly(c.dv, tile_bounds);
+
+				if (c.dv.size() == 0) {
+					clipped_to_nothing = true;
+					break;
+				}
 			}
 		}
 	}
 
-	json_object *json_filter = NULL;
+	std::string out;
-	if (filter.size() > 0) {
-		json_filter = parse_filter(filter.c_str());
-	}
 
-	for (auto const &s : sources) {
+	if (!clipped_to_nothing) {
-		std::string tile;
+		json_object *json_filter = NULL;
-		char buf[1000];
+		if (filter.size() > 0) {
-		int len;
+			json_filter = parse_filter(filter.c_str());
-
-		FILE *f = fopen(s.tile.c_str(), "rb");
-		if (f == NULL) {
-			perror(s.tile.c_str());
-			exit(EXIT_FAILURE);
 		}
 
-		while ((len = fread(buf, sizeof(char), 1000, f)) > 0) {
+		for (auto const &s : sources) {
-			tile.append(std::string(buf, len));
+			std::string tile;
+			char buf[1000];
+			int len;
+
+			FILE *f = fopen(s.tile.c_str(), "rb");
+			if (f == NULL) {
+				perror(s.tile.c_str());
+				exit(EXIT_FAILURE);
+			}
+
+			while ((len = fread(buf, sizeof(char), 1000, f)) > 0) {
+				tile.append(std::string(buf, len));
+			}
+			fclose(f);
+
+			input_tile t = s;
+			t.tile = std::move(tile);
+			its.push_back(std::move(t));
 		}
-		fclose(f);
 
-		input_tile t = s;
+		out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
-		t.tile = std::move(tile);
-		its.push_back(std::move(t));
 	}
 
-	std::string out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX, clipbboxes);
-
 	FILE *f = fopen(outfile, "wb");
 	if (f == NULL) {
 		perror(outfile);

platform.cpp

@@ -0,0 +1,56 @@
+#include "platform.hpp"
+#include <cstdlib>
+#include <cstdio>
+#include <unistd.h>
+#include <sys/resource.h>
+
+#if defined(__APPLE__) || defined(__FreeBSD__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/param.h>
+#include <sys/mount.h>
+#endif
+
+#include "errors.hpp"
+
+long get_num_avail_cpus() {
+	return sysconf(_SC_NPROCESSORS_ONLN);
+}
+
+long get_page_size() {
+	return sysconf(_SC_PAGESIZE);
+}
+
+size_t calc_memsize() {
+	size_t mem;
+
+#ifdef __APPLE__
+	int64_t hw_memsize;
+	size_t len = sizeof(int64_t);
+	if (sysctlbyname("hw.memsize", &hw_memsize, &len, NULL, 0) < 0) {
+		perror("sysctl hw.memsize");
+		exit(EXIT_MEMORY);
+	}
+	mem = hw_memsize;
+#else
+	long long pagesize = sysconf(_SC_PAGESIZE);
+	long long pages = sysconf(_SC_PHYS_PAGES);
+	if (pages < 0 || pagesize < 0) {
+		perror("sysconf _SC_PAGESIZE or _SC_PHYS_PAGES");
+		exit(EXIT_MEMORY);
+	}
+
+	mem = (long long) pages * pagesize;
+#endif
+
+	return mem;
+}
+
+size_t get_max_open_files() {
+	struct rlimit rl;
+	if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
+		perror("getrlimit");
+		exit(EXIT_PTHREAD);
+	}
+	return rl.rlim_cur;
+}

platform.hpp

@@ -0,0 +1,18 @@
+#ifndef PLATFORM_HPP
+#define PLATFORM_HPP
+
+#include <cstddef>
+
+long get_num_avail_cpus();
+
+long get_page_size();
+
+size_t calc_memsize();
+
+size_t get_max_open_files();
+
+constexpr const char *get_null_device() {
+	return "/dev/null";
+}
+
+#endif	// PLATFORM_HPP

tests/pbf/squirrels-clip.json

@@ -0,0 +1 @@
+{"coordinates":[[[-73.9722549,40.7904659],[-73.9755817,40.7674848],[-73.938783,40.767022],[-73.9457082,40.7791562],[-73.9628855,40.7760201],[-73.9629534,40.7914425],[-73.9722549,40.7904659]]],"crs":{"properties":{"name":"EPSG:4326"},"type":"name"},"type":"Polygon"}

tile-join.cpp

@@ -43,6 +43,7 @@
 #include "errors.hpp"
 #include "geometry.hpp"
 #include "thread.hpp"
+#include "platform.hpp"
 
 int pk = false;
 int pC = false;
@@ -1325,7 +1326,7 @@ int main(int argc, char **argv) {
 
 	struct tileset_reader *readers = NULL;
 
-	CPUS = sysconf(_SC_NPROCESSORS_ONLN);
+	CPUS = get_num_avail_cpus();
 
 	const char *TIPPECANOE_MAX_THREADS = getenv("TIPPECANOE_MAX_THREADS");
 	if (TIPPECANOE_MAX_THREADS != NULL) {

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.72.0"
+#define VERSION "v2.73.0"
 
 #endif