diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 16:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add -T option to coerce the types of feature attributes

Browse Source
This commit is contained in:
Eric Fischer
2017-04-17 15:20:03 -07:00
parent 65ee2cf0a5
commit 178b5d0054
9 changed files with 275 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+46 -10
View File
@@ -372,7 +372,7 @@ void *run_sort(void *v) {
return NULL;
}
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma) {
void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > *layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types) {
long long segs[CPUS + 1];
segs[0] = 0;
segs[CPUS] = len;
@@ -430,6 +430,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const
pja[i].layermap = &(*layermaps)[i];
pja[i].layername = &layername;
pja[i].uses_gamma = uses_gamma;
pja[i].attribute_types = attribute_types;
if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) {
perror("pthread_create");
@@ -473,6 +474,7 @@ struct read_parallel_arg {
unsigned *initial_y;
std::string layername;
bool uses_gamma;
std::map<std::string, int> const *attribute_types;
};
void *run_read_parallel(void *v) {
@@ -494,7 +496,7 @@ void *run_read_parallel(void *v) {
}
madvise(map, rpa->len, MADV_RANDOM); // sequential, but from several pointers at once
do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->reader, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->fname, rpa->basezoom, rpa->source, rpa->nlayers, rpa->layermaps, rpa->droprate, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma);
do_read_parallel(map, rpa->len, rpa->offset, rpa->reading, rpa->reader, rpa->progress_seq, rpa->exclude, rpa->include, rpa->exclude_all, rpa->fname, rpa->basezoom, rpa->source, rpa->nlayers, rpa->layermaps, rpa->droprate, rpa->initialized, rpa->initial_x, rpa->initial_y, rpa->maxzoom, rpa->layername, rpa->uses_gamma, rpa->attribute_types);
madvise(map, rpa->len, MADV_DONTNEED);
if (munmap(map, rpa->len) != 0) {
@@ -511,7 +513,7 @@ void *run_read_parallel(void *v) {
return NULL;
}
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma) {
void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, bool &parser_created, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, std::vector<std::map<std::string, layermap_entry> > &layermaps, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, int maxzoom, std::string layername, bool uses_gamma, std::map<std::string, int> const *attribute_types) {
// This has to kick off an intermediate thread to start the parser threads,
// so the main thread can get back to reading the next input stage while
// the intermediate thread waits for the completion of the parser threads.
@@ -548,6 +550,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i
rpa->maxzoom = maxzoom;
rpa->layername = layername;
rpa->uses_gamma = uses_gamma;
rpa->attribute_types = attribute_types;
if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) {
perror("pthread_create");
@@ -1006,7 +1009,7 @@ void choose_first_zoom(long long *file_bbox, struct reader *reader, unsigned *iz
}
}
int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description, bool guess_maxzoom) {
int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minzoom, int basezoom, double basezoom_marker_width, sqlite3 *outdb, const char *outdir, std::set<std::string> *exclude, std::set<std::string> *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, int read_parallel, int forcetable, const char *attribution, bool uses_gamma, long long *file_bbox, const char *description, bool guess_maxzoom, std::map<std::string, int> const *attribute_types) {
int ret = EXIT_SUCCESS;
struct reader reader[CPUS];
@@ -1211,7 +1214,7 @@ int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minz
}
if (map != NULL && map != MAP_FAILED) {
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma);
do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, &layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, uses_gamma, attribute_types);
overall_offset += st.st_size - off;
checkdisk(reader, CPUS);
@@ -1279,7 +1282,7 @@ int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minz
}
fflush(readfp);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types);
initial_offset += ahead;
overall_offset += ahead;
@@ -1316,7 +1319,7 @@ int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minz
fflush(readfp);
if (ahead > 0) {
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0);
start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, &parallel_parser, parser_created, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, layer, nlayers, layermaps, droprate, initialized, initial_x, initial_y, maxzoom, sources[layer].layer, gamma != 0, attribute_types);
if (parser_created) {
if (pthread_join(parallel_parser, NULL) != 0) {
@@ -1333,7 +1336,7 @@ int read_input(std::vector<source> &sources, char *fname, int &maxzoom, int minz
long long layer_seq = overall_offset;
json_pull *jp = json_begin_file(fp);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, layer, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, maxzoom, &layermaps[0], sources[layer].layer, uses_gamma);
parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, layer, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, maxzoom, &layermaps[0], sources[layer].layer, uses_gamma, attribute_types);
json_end(jp);
overall_offset = layer_seq;
checkdisk(reader, CPUS);
@@ -1924,6 +1927,33 @@ static bool has_name(struct option *long_options, int *pl) {
return false;
}
void set_attribute_type(std::map<std::string, int> &attribute_types, const char *arg) {
const char *s = strchr(arg, ':');
if (s == NULL) {
fprintf(stderr, "-T%s option must be in the form -Tname:type\n", arg);
exit(EXIT_FAILURE);
}
std::string name = std::string(arg, s - arg);
std::string type = std::string(s + 1);
int t = -1;
if (type == "int") {
t = mvt_int;
} else if (type == "float") {
t = mvt_float;
} else if (type == "string") {
t = mvt_string;
} else if (type == "bool") {
t = mvt_bool;
} else {
fprintf(stderr, "Attribute type (%s) must be int, float, string, or bool\n", type.c_str());
exit(EXIT_FAILURE);
}
attribute_types.insert(std::pair<std::string, int>(name, t));
}
int main(int argc, char **argv) {
#ifdef MTRACE
mtrace();
@@ -1956,6 +1986,7 @@ int main(int argc, char **argv) {
bool guess_maxzoom = false;
std::set<std::string> exclude, include;
std::map<std::string, int> attribute_types;
int exclude_all = 0;
int read_parallel = 0;
int files_open_at_start;
@@ -1991,6 +2022,7 @@ int main(int argc, char **argv) {
{"projection", required_argument, 0, 's'},
{"simplification", required_argument, 0, 'S'},
{"maximum-tile-bytes", required_argument, 0, 'M'},
{"attribute-type", required_argument, 0, 'T'},
{"exclude-all", no_argument, 0, 'X'},
{"force", no_argument, 0, 'f'},
@@ -2048,7 +2080,7 @@ int main(int argc, char **argv) {
}
}
while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:", long_options, NULL)) != -1) {
while ((i = getopt_long(argc, argv, "n:l:z:Z:B:d:D:m:o:e:x:y:r:b:t:g:p:a:XfFqvPL:A:s:S:M:N:T:", long_options, NULL)) != -1) {
switch (i) {
case 0:
break;
@@ -2244,6 +2276,10 @@ int main(int argc, char **argv) {
max_tile_size = atoll(optarg);
break;
case 'T':
set_attribute_type(attribute_types, optarg);
break;
default: {
int width = 7 + strlen(argv[0]);
fprintf(stderr, "Unknown option -%c\n", i);
@@ -2366,7 +2402,7 @@ int main(int argc, char **argv) {
long long file_bbox[4] = {UINT_MAX, UINT_MAX, 0, 0};
ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description, guess_maxzoom);
ret = read_input(sources, name ? name : out_mbtiles ? out_mbtiles : out_directory, maxzoom, minzoom, basezoom, basezoom_marker_width, outdb, out_directory, &exclude, &include, exclude_all, droprate, buffer, tmpdir, gamma, read_parallel, forcetable, attribution, gamma != 0, file_bbox, description, guess_maxzoom, &attribute_types);
if (outdb != NULL) {
mbtiles_close(outdb, argv);