diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Make file positions and lengths thread-safe

Browse Source
This commit is contained in:
Eric Fischer
2018-05-07 14:42:49 -07:00
parent 2cf0524037
commit 59dd095607
10 changed files with 110 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+13 -10
View File
@@ -287,7 +287,7 @@ int calc_feature_minzoom(struct index *ix, struct drop_state *ds, int maxzoom, d
return feature_minzoom;
}
static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, FILE *indexfile, int bytes, char *geom_map, FILE *geom_out, long long *geompos, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, double gamma, struct drop_state *ds) {
static void merge(struct mergelist *merges, size_t nmerges, unsigned char *map, FILE *indexfile, int bytes, char *geom_map, FILE *geom_out, std::atomic<long long> *geompos, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, double gamma, struct drop_state *ds) {
struct mergelist *head = NULL;
for (size_t i = 0; i < nmerges; i++) {
@@ -592,7 +592,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, std::atomi
parser_created = true;
}
void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, long long *geompos_out, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) {
void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int splits, long long mem, const char *tmpdir, long long *availfiles, FILE *geomfile, FILE *indexfile, std::atomic<long long> *geompos_out, long long *progress, long long *progress_max, long long *progress_reported, int maxzoom, int basezoom, double droprate, double gamma, struct drop_state *ds) {
// Arranged as bits to facilitate subdividing again if a subdivided file is still huge
int splitbits = log(splits) / log(2);
splits = 1 << splitbits;
@@ -601,7 +601,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
FILE *indexfiles[splits];
int geomfds[splits];
int indexfds[splits];
long long sub_geompos[splits];
std::atomic<long long> sub_geompos[splits];
int i;
for (i = 0; i < splits; i++) {
@@ -741,7 +741,7 @@ void radix1(int *geomfds_in, int *indexfds_in, int inputs, int prefix, int split
if (indexst.st_size > 0) {
if (indexst.st_size + geomst.st_size < mem) {
long long indexpos = indexst.st_size;
std::atomic<long long> indexpos(indexst.st_size);
int bytes = sizeof(struct index);
int page = sysconf(_SC_PAGESIZE);
@@ -917,7 +917,7 @@ void prep_drop_states(struct drop_state *ds, int maxzoom, int basezoom, double d
}
}
void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, long long *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
void radix(std::vector<struct reader> &readers, int nreaders, FILE *geomfile, FILE *indexfile, const char *tmpdir, std::atomic<long long> *geompos, int maxzoom, int basezoom, double droprate, double gamma) {
// Run through the index and geometry for each reader,
// splitting the contents out by index into as many
// sub-files as we can write to simultaneously.
@@ -1689,8 +1689,8 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
unlink(metaname);
long long metapos = 0;
long long poolpos = 0;
std::atomic<long long> metapos(0);
std::atomic<long long> poolpos(0);
for (size_t i = 0; i < CPUS; i++) {
if (readers[i].metapos > 0) {
@@ -1795,7 +1795,7 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
iy = justy;
}
long long geompos = 0;
std::atomic<long long> geompos(0);
/* initial tile is 0/0/0 */
serialize_int(geomfile, iz, &geompos, fname);
@@ -1821,13 +1821,16 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
perror("stat index");
exit(EXIT_FAILURE);
}
long long indexpos = indexst.st_size;
std::atomic<long long> indexpos(indexst.st_size);
progress_seq = indexpos / sizeof(struct index);
last_progress = 0;
if (!quiet) {
long long s = progress_seq;
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos, metapos, poolpos);
long long geompos_print = geompos;
long long metapos_print = metapos;
long long poolpos_print = poolpos;
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", s, geompos_print, metapos_print, poolpos_print);
}
if (indexpos == 0) {