Merge pull request #51 from mapbox/fitsample

Flag to drop as many features as necessary to keep tiles under the size limit

README.md

@@ -90,6 +90,7 @@ Options
  * -pk: Don't limit tiles to 500K bytes
  * -po: Don't reorder features to put the same properties in sequence
  * -pl: Let "dot" simplification apply to lines too
+ * -pd: Dynamically drop some fraction of features from large tiles to keep them under the 500K size limit. It will probably look ugly at the tile boundaries.
 
 Example
 -------

geojson.c

@@ -342,7 +342,7 @@ static void merge(struct merge *merges, int nmerges, unsigned char *map, FILE *f
 	}
 
 	while (head != NULL) {
-		fwrite(map + head->start, bytes, 1, f);
+		fwrite_check(map + head->start, bytes, 1, f, "merge temporary");
 		head->start += bytes;
 
 		struct merge *m = head;

tile.cc

@@ -354,6 +354,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 	int line_detail;
 	static bool evaluated = false;
 	double oprogress = 0;
+	double fraction = 1;
 
 	char *og = *geoms;
 
@@ -381,6 +382,8 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 			interval = exp(log(droprate) * (basezoom - z));
 		}
 
+		double fraction_accum = 0;
+
 		unsigned long long previndex = 0;
 		double scale = (double) (1LL << (64 - 2 * (z + 8)));
 		double gap = 0;
@@ -440,7 +443,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 				geom = remove_noop(geom, t);
 			}
 
-			if (line_detail == detail) { /* only write out the next zoom once, even if we retry */
+			if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */
 				if (geom.size() > 0 && z + 1 <= file_maxzoom) {
 					int j;
 					for (j = 0; j < 4; j++) {
@@ -552,6 +555,12 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 				}
 			}
 
+			fraction_accum += fraction;
+			if (fraction_accum < 1) {
+				continue;
+			}
+			fraction_accum -= 1;
+
 			bool reduced = false;
 			if (t == VT_POLYGON) {
 				geom = reduce_tiny_poly(geom, z, line_detail, &reduced, &accum_area);
@@ -611,7 +620,7 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 		}
 
 		for (j = 0; j < nlayers; j++) {
-			if (!prevent['o'] & 0xFF) {
+			if (!prevent['o' & 0xFF]) {
 				std::sort(features[j].begin(), features[j].end());
 			}
 
@@ -681,6 +690,15 @@ long long write_tile(char **geoms, char *metabase, unsigned *file_bbox, int z, u
 					evaluate(features[0], metabase, file_keys[0], layername, line_detail, compressed.size()); // XXX layer
 #endif
 				}
+
+				if (prevent['d' & 0xFF]) {
+					// The 95% is a guess to avoid too many retries
+					// and probably actually varies based on how much duplicated metadata there is
+
+					fraction = fraction * 500000 / compressed.size() * 0.95;
+					fprintf(stderr, "Going to try keeping %0.2f%% of the features to make it fit\n", fraction * 100);
+					line_detail++; // to keep it the same when the loop decrements it
+				}
 			} else {
 				mbtiles_write_tile(outdb, z, tx, ty, compressed.data(), compressed.size());
 				return count;