diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use std::set to track the layer-wide feature attribute types.

Browse Source 
Track them during parsing, not tiling.  Remove the old string pool code.
This commit is contained in:
Eric Fischer
2016-04-28 14:59:58 -07:00
parent 87b90a5033
commit adc70341ad
11 changed files with 108 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+25 -18
View File
@@ -39,11 +39,11 @@ extern "C" {
#include "tile.hpp"
#include "pool.hpp"
#include "mbtiles.hpp"
#include "projection.hpp"
#include "version.hpp"
#include "memfile.hpp"
#include "serial.hpp"
#include "mbtiles.hpp"
#include "main.hpp"
#include "geojson.hpp"
#include "geometry.hpp"
@@ -298,7 +298,7 @@ void *run_sort(void *v) {
return NULL;
}
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) {
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys) {
long long segs[CPUS + 1];
segs[0] = 0;
segs[CPUS] = len;
@@ -321,6 +321,11 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
struct parse_json_args pja[CPUS];
pthread_t pthreads[CPUS];
std::vector<std::set<type_and_string> > file_subkeys;
for (i = 0; i < CPUS; i++) {
file_subkeys.push_back(std::set<type_and_string>());
}
for (i = 0; i < CPUS; i++) {
pja[i].jp = json_begin_map(map + segs[i], segs[i + 1] - segs[i]);
@@ -348,6 +353,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
pja[i].initial_x = &initial_x[i];
pja[i].initial_y = &initial_y[i];
pja[i].readers = reader;
pja[i].file_keys = &file_subkeys[i];
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
perror("pthread_create");
@@ -362,6 +368,11 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
perror("pthread_join");
}
std::set<type_and_string>::iterator j;
for (j = file_subkeys[i].begin(); j != file_subkeys[i].end(); ++j) {
file_keys->insert(*j);
}
free(pja[i].jp->source);
json_end(pja[i].jp);
}
@@ -389,6 +400,7 @@ struct read_parallel_arg {
int *initialized;
unsigned *initial_x;
unsigned *initial_y;
std::set<type_and_string> *file_keys;
};
void *run_read_parallel(void *v) {
@@ -410,7 +422,7 @@ void *run_read_parallel(void *v) {
}
madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y);
do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys);
madvise(map, a->len, MADV_DONTNEED);
if (munmap(map, a->len) != 0) {
@@ -427,7 +439,7 @@ void *run_read_parallel(void *v) {
return NULL;
}
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y) {
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set<type_and_string> *file_keys) {
// This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads.
@@ -460,6 +472,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
rpa->initialized = initialized;
rpa->initial_x = initial_x;
rpa->initial_y = initial_y;
rpa->file_keys = file_keys;
if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
perror("pthread_create");
@@ -966,10 +979,12 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
nsources = 1;
}
std::vector<std::set<type_and_string> >file_keys;
long overall_offset = 0;
int source;
for (source = 0; source < nsources; source++) {
file_keys.push_back(std::set<type_and_string>());
std::string reading;
int fd;
@@ -1003,7 +1018,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
if (map != NULL && map != MAP_FAILED) {
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
overall_offset += st.st_size - off;
checkdisk(reader, CPUS);
@@ -1069,7 +1084,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
initial_offset += ahead;
overall_offset += ahead;
@@ -1105,7 +1120,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
fflush(readfp);
if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]);
if (pthread_join(parallel_parser, NULL) != 0) {
perror("pthread_join");
@@ -1119,7 +1134,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
long long layer_seq = overall_offset;
json_pull *jp = json_begin_file(fp);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0]);
json_end(jp);
overall_offset = layer_seq;
checkdisk(reader, CPUS);
@@ -1162,13 +1177,6 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
}
struct pool file_keys1[nlayers];
struct pool *file_keys[nlayers];
for (i = 0; i < nlayers; i++) {
pool_init(&file_keys1[i], 0);
file_keys[i] = &file_keys1[i];
}
char *layernames[nlayers];
for (i = 0; i < nlayers; i++) {
if (layername != NULL) {
@@ -1598,7 +1606,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
unsigned midx = 0, midy = 0;
int written = traverse_zooms(fd, size, meta, stringpool, file_keys, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y);
int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y);
if (maxzoom != written) {
fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
@@ -1615,7 +1623,7 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
}
if (poolpos > 0) {
madvise((void *) pool, poolpos, MADV_DONTNEED);
madvise((void *) stringpool, poolpos, MADV_DONTNEED);
if (munmap(stringpool, poolpos) != 0) {
perror("munmap stringpool");
}
@@ -1685,7 +1693,6 @@ int read_input(std::vector<source> &sources, char *fname, const char *layername,
mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers, forcetable, attribution);
for (i = 0; i < nlayers; i++) {
pool_free_strings(&file_keys1[i]);
free(layernames[i]);
}