diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Memory-map geobuf input instead of reading it into a temporary string

Browse Source
This commit is contained in:
Eric Fischer
2017-08-28 11:30:30 -07:00
parent 5943c82457
commit 7c031a9796
3 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
geobuf.cpp
+2 -2
View File
@@ -313,8 +313,8 @@ void readFeatureCollection(protozero::pbf_reader &pbf, size_t dim, double e, std
} }
} }
void parse_geobuf(struct serialization_state *sst, std::string const &src, int layer, std::string layername) { void parse_geobuf(struct serialization_state *sst, const char *src, size_t len, int layer, std::string layername) {
protozero::pbf_reader pbf(src); protozero::pbf_reader pbf(src, len);
size_t dim = 2; size_t dim = 2;
double e = 1e6; double e = 1e6;
geobuf.hpp
+1 -1
View File
@@ -8,6 +8,6 @@
#include "mbtiles.hpp" #include "mbtiles.hpp"
#include "serial.hpp" #include "serial.hpp"
void parse_geobuf(struct serialization_state *sst, std::string const &s, int layer, std::string layername); void parse_geobuf(struct serialization_state *sst, const char *s, size_t len, int layer, std::string layername);
#endif #endif
main.cpp
+19 -9
View File
@@ -1194,16 +1194,17 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
size_t layer = a->second.id; size_t layer = a->second.id;
if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".pbf")) { if (sources[source].file.size() > 4 && sources[source].file.substr(sources[source].file.size() - 4) == std::string(".pbf")) {
std::string s = ""; struct stat st;
if (fstat(fd, &st) != 0) {
char buf[20000]; perror("fstat");
ssize_t n; perror(sources[source].file.c_str());
while ((n = read(fd, buf, 20000)) > 0) { exit(EXIT_FAILURE);
s.append(std::string(buf, n));
} }
if (close(fd) != 0) { char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
perror("close"); if (map == MAP_FAILED) {
perror("mmap");
perror(sources[source].file.c_str());
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@@ -1232,7 +1233,16 @@ int read_input(std::vector<source> &sources, char *fname, int maxzoom, int minzo
sst.basezoom = basezoom; sst.basezoom = basezoom;
sst.attribute_types = attribute_types; sst.attribute_types = attribute_types;
parse_geobuf(&sst, s, layer, sources[layer].layer); parse_geobuf(&sst, map, st.st_size, layer, sources[layer].layer);
if (munmap(map, st.st_size) != 0) {
perror("munmap source file");
exit(EXIT_FAILURE);
}
if (close(fd) != 0) {
perror("close");
exit(EXIT_FAILURE);
}
overall_offset = layer_seq; overall_offset = layer_seq;
checkdisk(reader, CPUS); checkdisk(reader, CPUS);