From 834c05038c0d0a0a5c3d2fa0b42ba69d7ce04215 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Tue, 13 Mar 2018 14:51:41 -0700 Subject: [PATCH] Add --progress-interval setting to reduce progress indicator frequency --- CHANGELOG.md | 4 ++++ README.md | 1 + main.cpp | 43 ++++++++++++++++++++++++++++++++++++++----- main.hpp | 3 +++ man/tippecanoe.1 | 2 ++ serial.cpp | 2 +- tile.cpp | 6 ++++-- version.hpp | 2 +- 8 files changed, 54 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1497239d..a033c4b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.27.10 + +* Add --progress-interval setting to reduce progress indicator frequency + ## 1.27.9 * Make clusters look better by averaging locations of clustered points diff --git a/README.md b/README.md index 5dc0a219..4ad3454c 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,7 @@ tippecanoe -z5 -o filtered.mbtiles -j '{ "ne_10m_admin_0_countries": [ "all", [ * `-q` or `--quiet`: Work quietly instead of reporting progress or warning messages * `-Q` or `--no-progress-indicator`: Don't report progress, but still give warnings + * `-U` _seconds_ or `--progress-interval=`_seconds_: Don't report progress more often than the specified number of _seconds_. * `-v` or `--version`: Report Tippecanoe's version number ### Filters diff --git a/main.cpp b/main.cpp index cdf00b0b..af2fd235 100644 --- a/main.cpp +++ b/main.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,8 @@ static int min_detail = 7; int quiet = 0; int quiet_progress = 0; +double progress_interval = 0; +volatile double last_progress = 0; int geometry_scale = 0; double simplification = 1; size_t max_tile_size = 500000; @@ -293,6 +296,8 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, } } + last_progress = 0; + while (head != NULL) { struct index ix = *((struct index *) (map + head->start)); long long pos = *geompos; @@ -303,7 +308,7 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, // Count this as an 75%-accomplishment, since we already 25%-counted it *progress += (ix.end - ix.start) * 3 / 4; - if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { + if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) { fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); *progress_reported = 100 * *progress / *progress_max; } @@ -673,7 +678,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split // Count this as a 25%-accomplishment, since we will copy again *progress += (ix.end - ix.start) / 4; - if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { + if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) { fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); *progress_reported = 100 * *progress / *progress_max; } @@ -845,7 +850,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split // Count this as an 75%-accomplishment, since we already 25%-counted it *progress += (ix.end - ix.start) * 3 / 4; - if (!quiet && !quiet_progress && 100 * *progress / *progress_max != *progress_reported) { + if (!quiet && !quiet_progress && progress_time() && 100 * *progress / *progress_max != *progress_reported) { fprintf(stderr, "Reordering geometry: %lld%% \r", 100 * *progress / *progress_max); *progress_reported = 100 * *progress / *progress_max; } @@ -1819,6 +1824,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo long long indexpos = indexst.st_size; progress_seq = indexpos / sizeof(struct index); + last_progress = 0; if (!quiet) { fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); } @@ -1856,7 +1862,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo long long nprogress = 100 * ip / indices; if (nprogress != progress) { progress = nprogress; - if (!quiet && !quiet_progress) { + if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, "Maxzoom: %lld%% \r", progress); } } @@ -1952,7 +1958,7 @@ int read_input(std::vector &sources, char *fname, int maxzoom, int minzo long long nprogress = 100 * ip / indices; if (nprogress != progress) { progress = nprogress; - if (!quiet && !quiet_progress) { + if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, "Base zoom/drop rate: %lld%% \r", progress); } } @@ -2436,6 +2442,7 @@ int main(int argc, char **argv) { {"Progress indicator", 0, 0, 0}, {"quiet", no_argument, 0, 'q'}, {"no-progress-indicator", no_argument, 0, 'Q'}, + {"progress-interval", required_argument, 0, 'U'}, {"version", no_argument, 0, 'v'}, {"", 0, 0, 0}, @@ -2686,6 +2693,10 @@ int main(int argc, char **argv) { quiet_progress = 1; break; + case 'U': + progress_interval = atof_require(optarg, "Progress interval"); + break; + case 'p': { char *cp; for (cp = optarg; *cp != '\0'; cp++) { @@ -2937,3 +2948,25 @@ FILE *fopen_oflag(const char *name, const char *mode, int oflag) { } return fdopen(fd, mode); } + +bool progress_time() { + if (progress_interval == 0.0) { + return true; + } + + struct timeval tv; + double now; + if (gettimeofday(&tv, NULL) != 0) { + fprintf(stderr, "%s: Can't get the time of day: %s\n", *av, strerror(errno)); + now = 0; + } else { + now = tv.tv_sec + tv.tv_usec / 1000000.0; + } + + if (now - last_progress >= progress_interval) { + last_progress = now; + return true; + } else { + return false; + } +} diff --git a/main.hpp b/main.hpp index b860670d..7e3c6475 100644 --- a/main.hpp +++ b/main.hpp @@ -22,6 +22,8 @@ void checkdisk(std::vector *r); extern int geometry_scale; extern int quiet; extern int quiet_progress; +extern double progress_interval; +extern volatile double last_progress; extern size_t CPUS; extern size_t TEMP_FILES; @@ -32,6 +34,7 @@ extern int cluster_distance; int mkstemp_cloexec(char *name); FILE *fopen_oflag(const char *name, const char *mode, int oflag); +bool progress_time(); #define MAX_ZOOM 24 diff --git a/man/tippecanoe.1 b/man/tippecanoe.1 index b3356336..acfe2375 100644 --- a/man/tippecanoe.1 +++ b/man/tippecanoe.1 @@ -355,6 +355,8 @@ If you don't specify, it will use \fB\fC/tmp\fR\&. .IP \(bu 2 \fB\fC\-Q\fR or \fB\fC\-\-no\-progress\-indicator\fR: Don't report progress, but still give warnings .IP \(bu 2 +\fB\fC\-U\fR \fIseconds\fP or \fB\fC\-\-progress\-interval=\fR\fIseconds\fP: Don't report progress more often than the specified number of \fIseconds\fP\&. +.IP \(bu 2 \fB\fC\-v\fR or \fB\fC\-\-version\fR: Report Tippecanoe's version number .RE .SS Filters diff --git a/serial.cpp b/serial.cpp index ef6fcb91..e11ca16d 100644 --- a/serial.cpp +++ b/serial.cpp @@ -633,7 +633,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { if (*(sst->progress_seq) % 10000 == 0) { checkdisk(sst->readers); - if (!quiet && !quiet_progress) { + if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, "Read %.2f million features\r", *sst->progress_seq / 1000000.0); } } diff --git a/tile.cpp b/tile.cpp index 94952acb..c37abc87 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1298,7 +1298,7 @@ serial_feature next_feature(FILE *geoms, long long *geompos_in, char *metabase, double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; if (progress >= *oprogress + 0.1) { - if (!quiet && !quiet_progress) { + if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); } *oprogress = progress; @@ -2163,7 +2163,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s double progress = floor(((((*geompos_in + *along - alongminus) / (double) todo) + (pass - (2 - passes))) / passes + z) / (maxzoom + 1) * 1000) / 10; if (progress >= oprogress + 0.1) { - if (!quiet && !quiet_progress) { + if (!quiet && !quiet_progress && progress_time()) { fprintf(stderr, " %3.1f%% %d/%u/%u \r", progress, z, tx, ty); } oprogress = progress; @@ -2477,6 +2477,8 @@ void *run_thread(void *vargs) { } int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, int &maxzoom, int minzoom, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, const char *tmpdir, double gamma, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, double simplification, std::vector> &layermaps, const char *prefilter, const char *postfilter, std::map const *attribute_accum) { + last_progress = 0; + // The existing layermaps are one table per input thread. // We need to add another one per *tiling* thread so that it can be // safely changed during tiling. diff --git a/version.hpp b/version.hpp index 88de22ff..9953eea9 100644 --- a/version.hpp +++ b/version.hpp @@ -1,6 +1,6 @@ #ifndef VERSION_HPP #define VERSION_HPP -#define VERSION "tippecanoe v1.27.9\n" +#define VERSION "tippecanoe v1.27.10\n" #endif