Compare commits

...
3 Commits
4 changed files with 77 additions and 16 deletions
+1
View File
@@ -3098,6 +3098,7 @@ int main(int argc, char **argv) {
{"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1},
{"cluster-densest-as-needed", no_argument, &additional[A_CLUSTER_DENSEST_AS_NEEDED], 1},
{"keep-point-cluster-position", no_argument, &additional[A_KEEP_POINT_CLUSTER_POSITION], 1},
{"distinguish-duplicates", no_argument, &additional[A_DISTINGUISH_DUPLICATES], 1},
{"Dropping tightly overlapping features", 0, 0, 0},
{"gamma", required_argument, 0, 'g'},
+1
View File
@@ -29,6 +29,7 @@
#define A_VISVALINGAM ((int) 'v')
#define A_DETECT_WRAPAROUND ((int) 'w')
#define A_KEEP_POINT_CLUSTER_POSITION ((int) 'a')
#define A_DISTINGUISH_DUPLICATES ((int) 'u')
#define P_TILE_COMPRESSION ((int) 'C')
#define P_DUPLICATION ((int) 'D')
+1
View File
@@ -743,6 +743,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::
prevent[P_DYNAMIC_DROP] ||
additional[A_INCREASE_GAMMA_AS_NEEDED] ||
additional[A_GENERATE_POLYGON_LABEL_POINTS] ||
additional[A_DISTINGUISH_DUPLICATES] ||
sst->uses_gamma ||
retain_points_multiplier > 1 ||
preserve_multiplier_density_threshold > 0 ||
+74 -16
View File
@@ -1054,6 +1054,10 @@ static bool skip_next_feature(decompressor *geoms, std::atomic<long long> *geomp
struct next_feature_state {
unsigned long long previndex = 0;
unsigned long long prev_not_dropped_index = 0;
std::vector<FILE *> deferrals;
size_t which_deferral = 0;
bool doing_deferrals = false;
};
// This function is called repeatedly from write_tile() to retrieve the next feature
@@ -1065,31 +1069,85 @@ static serial_feature next_feature(decompressor *geoms, std::atomic<long long> *
while (1) {
serial_feature sf;
long long len;
std::string s;
if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
fprintf(stderr, "Unexpected physical EOF in feature stream\n");
exit(EXIT_READ);
}
if (len <= 0) {
if (compressed) {
geoms->end(geompos_in);
if (next_feature_state.doing_deferrals) {
if (next_feature_state.which_deferral >= next_feature_state.deferrals.size()) {
// done with direct deserialization, done with deferred deserialization
for (auto const fp : next_feature_state.deferrals) {
fclose(fp);
}
sf.t = -2;
return sf;
}
sf.t = -2;
return sf;
}
if (fread(&len, sizeof(len), 1, next_feature_state.deferrals[next_feature_state.which_deferral]) != 1) {
next_feature_state.which_deferral++;
continue;
}
std::string s;
s.resize(len);
size_t n = geoms->fread((void *) s.c_str(), sizeof(char), s.size(), geompos_in);
if (n != s.size()) {
fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size());
exit(EXIT_READ);
s.resize(len);
if (fread((void *) s.c_str(), 1, len, next_feature_state.deferrals[next_feature_state.which_deferral]) != (size_t) len) {
fprintf(stderr, "short read in deferred deserialization: %s\n", strerror(errno));
exit(EXIT_READ);
}
} else {
if (geoms->deserialize_long_long(&len, geompos_in) == 0) {
fprintf(stderr, "Unexpected physical EOF in feature stream\n");
exit(EXIT_READ);
}
if (len <= 0) {
if (compressed) {
geoms->end(geompos_in);
}
next_feature_state.doing_deferrals = true;
next_feature_state.which_deferral = 0;
for (auto const fp : next_feature_state.deferrals) {
rewind(fp);
}
continue;
}
s.resize(len);
size_t n = geoms->fread((void *) s.c_str(), sizeof(char), s.size(), geompos_in);
if (n != s.size()) {
fprintf(stderr, "Short read (%zu for %zu) from geometry\n", n, s.size());
exit(EXIT_READ);
}
}
sf = deserialize_feature(s, z, tx, ty, initial_x, initial_y);
sf.stringpool = global_stringpool + pool_off[sf.segment];
if (!next_feature_state.doing_deferrals && additional[A_DISTINGUISH_DUPLICATES] && sf.index == next_feature_state.previndex) {
if (next_feature_state.which_deferral >= next_feature_state.deferrals.size()) {
std::string tmpname = "/tmp/XXXXXXXXXX";
int fd = mkstemp((char *) tmpname.c_str());
if (fd < 0) {
fprintf(stderr, "Can't create temporary file %zu for feature deferral: %s\n", next_feature_state.which_deferral, strerror(errno));
exit(EXIT_OPEN);
}
FILE *fp = fdopen(fd, "wb+");
if (fp == NULL) {
fprintf(stderr, "Can't reopen temporary file for feature deferral: %s\n", strerror(errno));
exit(EXIT_OPEN);
}
next_feature_state.deferrals.resize(next_feature_state.which_deferral + 1);
next_feature_state.deferrals[next_feature_state.which_deferral] = fp;
}
fwrite(&len, sizeof(len), 1, next_feature_state.deferrals[next_feature_state.which_deferral]);
fwrite(s.c_str(), 1, len, next_feature_state.deferrals[next_feature_state.which_deferral]);
next_feature_state.which_deferral++;
continue;
} else {
next_feature_state.which_deferral = 0;
}
// with fractional zoom level, so we can target a specific number
// of features to keep with retain-points-multiplier, not just the
// powers of the drop rate