diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge per-thread layer names and file keys

Browse Source
This commit is contained in:
Eric Fischer
2016-08-29 16:38:57 -07:00
parent c26fa23564
commit 7f49ce5caa
7 changed files with 75 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+44 -8
View File
@@ -300,7 +300,7 @@ void *run_sort(void *v) {
return NULL;
}
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) {
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom, std::string layername) {
long long segs[CPUS + 1];
segs[0] = 0;
segs[CPUS] = len;
@@ -358,6 +358,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
pja[i].file_keys = &file_subkeys[i];
pja[i].maxzoom = maxzoom;
pja[i].layermap = &(*layermaps)[i];
pja[i].layername = &layername;
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
perror("pthread_create");
@@ -406,6 +407,7 @@ struct read_parallel_arg {
unsigned *initial_x;
unsigned *initial_y;
std::set<type_and_string> *file_keys;
std::string layername;
};
void *run_read_parallel(void *v) {
@@ -427,7 +429,7 @@ void *run_read_parallel(void *v) {
}
madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->layermaps, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys, a->maxzoom);
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->layermaps, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys, a->maxzoom, a->layername);
madvise(map, a->len, MADV_DONTNEED);
if (munmap(map, a->len) != 0) {
@@ -444,7 +446,7 @@ void *run_read_parallel(void *v) {
return NULL;
}
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom) {
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys, int maxzoom, std::string layername) {
// This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads.
@@ -480,6 +482,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
rpa->initial_y = initial_y;
rpa->file_keys = file_keys;
rpa->maxzoom = maxzoom;
rpa->layername = layername;
if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
perror("pthread_create");
@@ -862,6 +865,37 @@ void radix(struct reader *reader, int nreaders, FILE *geomfile, int geomfd, FILE
}
}
std::map<std::string, layermap_entry> merge_layermaps(std::vector<std::map<std::string, layermap_entry> > const &maps) {
std::map<std::string, layermap_entry> out;
for (size_t i = 0; i < maps.size(); i++) {
for (auto map = maps[i].begin(); map != maps[i].end(); ++map) {
if (out.count(map->first) == 0) {
out.insert(std::pair<std::string, layermap_entry>(map->first, layermap_entry(out.size())));
}
auto out_entry = out.find(map->first);
if (out_entry == out.end()) {
fprintf(stderr, "Internal error merging layers\n");
exit(EXIT_FAILURE);
}
for (auto fk = map->second.file_keys.begin(); fk != map->second.file_keys.end(); ++fk) {
out_entry->second.file_keys.insert(*fk);
}
if (additional[A_CALCULATE_FEATURE_DENSITY]) {
type_and_string tas;
tas.type = VT_NUMBER;
tas.string = "tippecanoe_feature_density";
out_entry->second.file_keys.insert(tas);
}
}
}
return out;
}
int read_input(std::vector<source> &sources, char *fname, const char *layername, int maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution) {
int ret = EXIT_SUCCESS;
@@ -1088,7 +1122,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
if (map != NULL && map != MAP_FAILED) {
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]);
overall_offset += st.st_size - off;
checkdisk(reader, CPUS);
@@ -1156,7 +1190,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]);
initial_offset += ahead;
overall_offset += ahead;
@@ -1193,7 +1227,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
fflush(readfp);
if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, layermaps, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom, layernames[source < nlayers ? source : 0]);
if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) {
@@ -1210,7 +1244,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
long long layer_seq = overall_offset;
json_pull *jp = json_begin_file(fp);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0], maxzoom, &layermaps[0]);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0], maxzoom, &layermaps[0], layernames[source < nlayers ? source : 0]);
json_end(jp);
overall_offset = layer_seq;
checkdisk(reader, CPUS);
@@ -1711,7 +1745,9 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
midlon = maxlon;
}
mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution);
std::map<std::string, layermap_entry> merged_lm = merge_layermaps(layermaps);
mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution, merged_lm);
return ret;
}