Add a way to run tippecanoe single-threaded for profiling

This commit is contained in:
Erica Fischer
2024-02-01 11:38:48 -08:00
parent 6a2bce8164
commit aa96f126c1
9 changed files with 43 additions and 12 deletions
+2 -2
View File
@@ -58,7 +58,7 @@ C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I.
LIBS = -L/usr/local/lib
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o compression.o clip.o sort.o attribute.o thread.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-enumerate: enumerate.o
@@ -67,7 +67,7 @@ tippecanoe-enumerate: enumerate.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3
tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o
tile-join: tile-join.o projection.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o clip.o attribute.o thread.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
+2 -1
View File
@@ -7,6 +7,7 @@
#include "milo/dtoa_milo.h"
#include "main.hpp"
#include "errors.hpp"
#include "thread.hpp"
static constexpr uint8_t magicbytes[8] = { 0x66, 0x67, 0x62, 0x03, 0x66, 0x67, 0x62, 0x01 };
@@ -303,7 +304,7 @@ void fgbRunQueue() {
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, fgb_run_parse_feature, &qra[i]) != 0) {
if (thread_create(&pthreads[i], NULL, fgb_run_parse_feature, &qra[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
+2 -1
View File
@@ -15,6 +15,7 @@
#include "jsonpull/jsonpull.h"
#include "text.hpp"
#include "errors.hpp"
#include "thread.hpp"
#define POINT 0
#define MULTIPOINT 1
@@ -465,7 +466,7 @@ void runQueue() {
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, run_parse_feature, &qra[i]) != 0) {
if (thread_create(&pthreads[i], NULL, run_parse_feature, &qra[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
+4 -3
View File
@@ -67,6 +67,7 @@
#include "read_json.hpp"
#include "sort.hpp"
#include "attribute.hpp"
#include "thread.hpp"
static int low_detail = 12;
static int full_detail = -1;
@@ -518,7 +519,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
if (thread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
@@ -735,7 +736,7 @@ void start_parsing(int fd, STREAM *fp, long long offset, long long len, std::ato
rpa->want_dist = want_dist;
rpa->filters = filters;
if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
if (thread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
@@ -931,7 +932,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
}
for (size_t a = 0; a < CPUS; a++) {
if (pthread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) {
if (thread_create(&pthreads[a], NULL, run_sort, &args[a]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
+2 -1
View File
@@ -25,6 +25,7 @@
#include "geometry.hpp"
#include "serial.hpp"
#include "errors.hpp"
#include "thread.hpp"
extern "C" {
#include "jsonpull/jsonpull.h"
@@ -643,7 +644,7 @@ std::vector<mvt_layer> filter_layers(const char *filter, std::vector<mvt_layer>
wa.extent = extent;
pthread_t writer;
if (pthread_create(&writer, NULL, run_writer, &wa) != 0) {
if (thread_create(&writer, NULL, run_writer, &wa) != 0) {
perror("pthread_create (filter writer)");
exit(EXIT_PTHREAD);
}
+22
View File
@@ -0,0 +1,22 @@
#include <stdlib.h>
#include "thread.hpp"
// It is harder to profile tippecanoe because of its normal use of multiple threads.
// If you set TIPPECANOE_NO_THREADS in the environment, it will run everything in
// the main thread instead for more straightforward profiling. (The threads will
// then still be created so they will be reaped, but they will immediately return.)
static const char *no_threads = getenv("TIPPECANOE_NO_THREADS");
static void *do_nothing(void *arg) {
return arg;
}
int thread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) {
if (no_threads != NULL) {
void *ret = start_routine(arg);
return pthread_create(thread, attr, do_nothing, ret);
} else {
return pthread_create(thread, attr, start_routine, arg);
}
}
+3
View File
@@ -0,0 +1,3 @@
#include <pthread.h>
int thread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg);
+2 -1
View File
@@ -42,6 +42,7 @@
#include "milo/dtoa_milo.h"
#include "errors.hpp"
#include "geometry.hpp"
#include "thread.hpp"
int pk = false;
int pC = false;
@@ -812,7 +813,7 @@ void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<
}
for (size_t i = 0; i < CPUS; i++) {
if (pthread_create(&pthreads[i], NULL, join_worker, &args[i]) != 0) {
if (thread_create(&pthreads[i], NULL, join_worker, &args[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
+4 -3
View File
@@ -45,6 +45,7 @@
#include "compression.hpp"
#include "protozero/varint.hpp"
#include "attribute.hpp"
#include "thread.hpp"
extern "C" {
#include "jsonpull/jsonpull.h"
@@ -2152,7 +2153,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
rpa.first_time = first_time;
rpa.compressed = compressed_input;
if (pthread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
if (thread_create(&prefilter_writer, NULL, run_prefilter, &rpa) != 0) {
perror("pthread_create (prefilter writer)");
exit(EXIT_PTHREAD);
}
@@ -2561,7 +2562,7 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
args[i].nodepos = nodepos;
if (tasks > 1) {
if (pthread_create(&pthreads[i], NULL, partial_feature_worker, &args[i]) != 0) {
if (thread_create(&pthreads[i], NULL, partial_feature_worker, &args[i]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}
@@ -3381,7 +3382,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *stringpool, std::atomic<
args[thread].shared_nodes_map = shared_nodes_map;
args[thread].nodepos = nodepos;
if (pthread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
if (thread_create(&pthreads[thread], NULL, run_thread, &args[thread]) != 0) {
perror("pthread_create");
exit(EXIT_PTHREAD);
}