Check how many CPUs and filehandles there are instead of guessing

geojson.c

@@ -17,6 +17,7 @@
 #include <limits.h>
 #include <sqlite3.h>
 #include <stdarg.h>
+#include <sys/resource.h>
 
 #include "jsonpull.h"
 #include "tile.h"
@@ -60,6 +61,27 @@ int mb_geometry[GEOM_TYPES] = {
 	VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON,
 };
 
+int CPUS;
+int TEMP_FILES;
+
+void init_cpus() {
+        CPUS = sysconf(_SC_NPROCESSORS_ONLN);
+        if (CPUS < 1) {
+                CPUS = 1;
+        }
+
+        TEMP_FILES = 64;
+        struct rlimit rl;
+        if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
+                perror("getrlimit");
+        } else {
+                TEMP_FILES = rl.rlim_cur / 3;
+                if (TEMP_FILES > CPUS * 4) {
+                        TEMP_FILES = CPUS * 4;
+                }
+        }
+}
+
 size_t fwrite_check(const void *ptr, size_t size, size_t nitems, FILE *stream, const char *fname) {
 	size_t w = fwrite(ptr, size, nitems, stream);
 	if (w != nitems) {
@@ -1167,6 +1189,8 @@ int main(int argc, char **argv) {
 	mtrace();
 #endif
 
+	init_cpus();
+
 	extern int optind;
 	extern char *optarg;
 	int i;

tile.cc

@@ -818,7 +818,7 @@ long long write_tile(char **geoms, char *metabase, char *stringpool, int z, unsi
 struct task {
 	int fileno;
 	struct task *next;
-} tasks[TEMP_FILES];
+};
 
 struct write_tile_args {
 	struct task *tasks;
@@ -961,9 +961,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 			}
 		}
 
-#define MAX_THREADS 20  // XXX Obtain from sysctl(hw.ncpu), /proc/cpuinfo, etc.
+		int threads = CPUS;
-
-		int threads = MAX_THREADS;
 		if (threads > TEMP_FILES / 4) {
 			threads = TEMP_FILES / 4;
 		}
@@ -978,6 +976,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo
 
 		// Assign temporary files to threads
 
+		struct task tasks[TEMP_FILES];
 		struct dispatch {
 			struct task *tasks;
 			long long todo;

tile.h

@@ -33,4 +33,5 @@ extern unsigned initial_x, initial_y;
 extern int geometry_scale;
 extern int quiet;
 
-#define TEMP_FILES 64
+extern int CPUS;
+extern int TEMP_FILES;