diff --git a/main.cpp b/main.cpp index 0447a649..09e831bc 100644 --- a/main.cpp +++ b/main.cpp @@ -327,8 +327,12 @@ static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, while (head != NULL) { struct index ix = *((struct index *) (map + head->start)); long long pos = *geompos; - fwrite_check(geom_map + ix.start, 1, ix.end - ix.start, geom_out, "merge geometry"); - *geompos += ix.end - ix.start; + + // MAGIC: This knows that the feature minzoom is the last byte of the serialized feature + // and is writing one byte less and then adding the byte for the minzoom. + + fwrite_check(geom_map + ix.start, 1, ix.end - ix.start - 1, geom_out, "merge geometry"); + *geompos += ix.end - ix.start - 1; int feature_minzoom = calc_feature_minzoom(&ix, ds, maxzoom, gamma); serialize_byte(geom_out, feature_minzoom, geompos, "merge geometry"); @@ -1950,7 +1954,7 @@ std::pair read_input(std::vector &sources, char *fname, i radix(readers, CPUS, geomfile, indexfile, tmpdir, &geompos, maxzoom, basezoom, droprate, gamma); /* end of tile */ - serialize_byte(geomfile, -2, &geompos, fname); + serialize_ulong_long(geomfile, 0, &geompos, fname); // EOF if (fclose(geomfile) != 0) { perror("fclose geom"); diff --git a/serial.cpp b/serial.cpp index e0972a6c..a2436016 100644 --- a/serial.cpp +++ b/serial.cpp @@ -234,8 +234,10 @@ static void write_geometry(drawvec const &dv, std::string &out, long long wx, lo } // called from generating the next zoom level -void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy, bool include_minzoom) { - serialize_byte(geomfile, sf->t, geompos, fname); +void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy) { + std::string s; + + serialize_byte(s, sf->t); #define FLAG_LAYER 7 @@ -257,63 +259,81 @@ void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomichas_tippecanoe_minzoom << FLAG_MINZOOM; layer |= sf->has_tippecanoe_maxzoom << FLAG_MAXZOOM; - serialize_long_long(geomfile, layer, geompos, fname); + serialize_long_long(s, layer); if (sf->seq != 0) { - serialize_long_long(geomfile, sf->seq, geompos, fname); + serialize_long_long(s, sf->seq); } if (sf->has_tippecanoe_minzoom) { - serialize_int(geomfile, sf->tippecanoe_minzoom, geompos, fname); + serialize_int(s, sf->tippecanoe_minzoom); } if (sf->has_tippecanoe_maxzoom) { - serialize_int(geomfile, sf->tippecanoe_maxzoom, geompos, fname); + serialize_int(s, sf->tippecanoe_maxzoom); } if (sf->has_id) { - serialize_ulong_long(geomfile, sf->id, geompos, fname); + serialize_ulong_long(s, sf->id); } - serialize_int(geomfile, sf->segment, geompos, fname); + serialize_int(s, sf->segment); std::string geom; write_geometry(sf->geometry, geom, wx, wy); - serialize_ulong_long(geomfile, geom.size(), geompos, fname); - fwrite_check(geom.c_str(), sizeof(char), geom.size(), geomfile, fname); - *geompos += geom.size(); + serialize_ulong_long(s, geom.size()); + s += geom; if (sf->index != 0) { - serialize_ulong_long(geomfile, sf->index, geompos, fname); + serialize_ulong_long(s, sf->index); } if (sf->label_point != 0) { - serialize_ulong_long(geomfile, sf->label_point, geompos, fname); + serialize_ulong_long(s, sf->label_point); } if (sf->extent != 0) { - serialize_long_long(geomfile, sf->extent, geompos, fname); + serialize_long_long(s, sf->extent); } - serialize_long_long(geomfile, sf->keys.size(), geompos, fname); + serialize_long_long(s, sf->keys.size()); for (size_t i = 0; i < sf->keys.size(); i++) { - serialize_long_long(geomfile, sf->keys[i], geompos, fname); - serialize_long_long(geomfile, sf->values[i], geompos, fname); + serialize_long_long(s, sf->keys[i]); + serialize_long_long(s, sf->values[i]); } - if (include_minzoom) { - serialize_byte(geomfile, sf->feature_minzoom, geompos, fname); - } + serialize_byte(s, sf->feature_minzoom); + + serialize_ulong_long(geomfile, s.size(), geompos, fname); + fwrite_check(s.c_str(), sizeof(char), s.size(), geomfile, fname); + *geompos += s.size(); } serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y) { serial_feature sf; + std::string s; + unsigned long long len; - deserialize_byte_io(geoms, &sf.t, geompos_in); - if (sf.t < 0) { + if (deserialize_ulong_long_io(geoms, &len, geompos_in) == 0) { + sf.t = -2; + return sf; + } + if (len == 0) { + sf.t = -2; return sf; } - deserialize_long_long_io(geoms, &sf.layer, geompos_in); + s.resize(len); + size_t n = fread((void *) s.c_str(), sizeof(char), len, geoms); + if (n != len) { + fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size()); + exit(EXIT_READ); + } + *geompos_in += n; + + char *cp = (char *) s.c_str(); + + deserialize_byte(&cp, &sf.t); + deserialize_long_long(&cp, &sf.layer); sf.seq = 0; if (sf.layer & (1 << FLAG_SEQ)) { - deserialize_long_long_io(geoms, &sf.seq, geompos_in); + deserialize_long_long(&cp, &sf.seq); } sf.tippecanoe_minzoom = -1; @@ -321,61 +341,58 @@ serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_ sf.id = 0; sf.has_id = false; if (sf.layer & (1 << FLAG_MINZOOM)) { - deserialize_int_io(geoms, &sf.tippecanoe_minzoom, geompos_in); + deserialize_int(&cp, &sf.tippecanoe_minzoom); } if (sf.layer & (1 << FLAG_MAXZOOM)) { - deserialize_int_io(geoms, &sf.tippecanoe_maxzoom, geompos_in); + deserialize_int(&cp, &sf.tippecanoe_maxzoom); } if (sf.layer & (1 << FLAG_ID)) { sf.has_id = true; - deserialize_ulong_long_io(geoms, &sf.id, geompos_in); + deserialize_ulong_long(&cp, &sf.id); } - deserialize_int_io(geoms, &sf.segment, geompos_in); + deserialize_int(&cp, &sf.segment); sf.index = 0; sf.label_point = 0; sf.extent = 0; unsigned long long geom_len; - deserialize_ulong_long_io(geoms, &geom_len, geompos_in); - - std::string geom; - geom.resize(geom_len); - size_t n = fread((void *) geom.c_str(), sizeof(char), geom_len, geoms); - if (n != geom_len) { - fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, geom.size()); - exit(EXIT_READ); - } - *geompos_in += geom_len; - - char *s = (char *) geom.c_str(); - sf.geometry = decode_geometry(&s, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]); + deserialize_ulong_long(&cp, &geom_len); + std::string geom(cp, geom_len); + cp += geom_len; + char *cp2 = (char *) geom.c_str(); + sf.geometry = decode_geometry(&cp2, z, tx, ty, sf.bbox, initial_x[sf.segment], initial_y[sf.segment]); if (sf.layer & (1 << FLAG_INDEX)) { - deserialize_ulong_long_io(geoms, &sf.index, geompos_in); + deserialize_ulong_long(&cp, &sf.index); } if (sf.layer & (1 << FLAG_LABEL_POINT)) { - deserialize_ulong_long_io(geoms, &sf.label_point, geompos_in); + deserialize_ulong_long(&cp, &sf.label_point); } if (sf.layer & (1 << FLAG_EXTENT)) { - deserialize_long_long_io(geoms, &sf.extent, geompos_in); + deserialize_long_long(&cp, &sf.extent); } sf.layer >>= FLAG_LAYER; long long count; - deserialize_long_long_io(geoms, &count, geompos_in); + deserialize_long_long(&cp, &count); for (long long i = 0; i < count; i++) { long long k, v; - deserialize_long_long_io(geoms, &k, geompos_in); - deserialize_long_long_io(geoms, &v, geompos_in); + deserialize_long_long(&cp, &k); + deserialize_long_long(&cp, &v); sf.keys.push_back(k); sf.values.push_back(v); } - deserialize_byte_io(geoms, &sf.feature_minzoom, geompos_in); + deserialize_byte(&cp, &sf.feature_minzoom); + + if (cp != s.c_str() + len) { + fprintf(stderr, "wrong length decoding feature: used %zd, len is %llu\n", cp - s.c_str(), len); + exit(EXIT_IMPOSSIBLE); + } return sf; } @@ -748,7 +765,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf) { long long geomstart = r->geompos; sf.geometry = scaled_geometry; - serialize_feature(r->geomfile, &sf, &r->geompos, sst->fname, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y)), false); + serialize_feature(r->geomfile, &sf, &r->geompos, sst->fname, SHIFT_RIGHT(*(sst->initial_x)), SHIFT_RIGHT(*(sst->initial_y))); struct index index; index.start = geomstart; diff --git a/serial.hpp b/serial.hpp index e521164c..25396188 100644 --- a/serial.hpp +++ b/serial.hpp @@ -75,7 +75,7 @@ struct serial_feature { bool dropped = false; }; -void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy, bool include_minzoom); +void serialize_feature(FILE *geomfile, serial_feature *sf, std::atomic *geompos, const char *fname, long long wx, long long wy); serial_feature deserialize_feature(FILE *geoms, std::atomic *geompos_in, unsigned z, unsigned tx, unsigned ty, unsigned *initial_x, unsigned *initial_y); struct reader { diff --git a/tile.cpp b/tile.cpp index 14f74fee..ca6559a9 100644 --- a/tile.cpp +++ b/tile.cpp @@ -422,7 +422,7 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u sf.values.push_back(metavals[i]); } - serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment]), true); + serialize_feature(geomfile[j], &sf, &geompos[j], fname, SHIFT_RIGHT(initial_x[segment]), SHIFT_RIGHT(initial_y[segment])); } } } @@ -2400,7 +2400,7 @@ long long write_tile(FILE *geoms, std::atomic *geompos_in, char *stri int j; for (j = 0; j < child_shards; j++) { if (within[j]) { - serialize_byte(geomfile[j], -2, &geompos[j], fname); + serialize_ulong_long(geomfile[j], 0, &geompos[j], fname); // EOF within[j] = 0; } }