diegoripley/tippecanoe
1
0
Fork 0
mirror of https://github.com/felt/tippecanoe.git synced 2026-10-02 08:25:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Reduce tippecanoe memory usage (#5)

Browse Source 
* Change JSON objects to a union type to use less memory

* Stop storing the string representation of JSON numbers

* Restore the ability to create features with large integer attributes

* Make sure large-integer feature IDs still behave as before

* Add missing #include

* Don't preallocate as much space for arrays and objects

* Treat inability to check free disk space as a warning, not an error

* Update changelog and version
This commit is contained in:
Erica Fischer
2022-08-09 15:18:51 -07:00
committed by GitHub
parent 6978a0e102
commit 67cd9d8d85
20 changed files with 911 additions and 827 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jsontool.cpp
+34 -23
View File
@@ -11,6 +11,7 @@
#include "csv.hpp"
#include "text.hpp"
#include "geojson-loop.hpp"
#include "milo/dtoa_milo.h"
int fail = EXIT_SUCCESS;
bool wrap = false;
@@ -147,7 +148,7 @@ void out(std::string const &s, int type, json_object *properties) {
if (o != NULL) {
found = true;
if (o->type == JSON_STRING || o->type == JSON_NUMBER) {
extracted = sort_quote(o->string);
extracted = sort_quote(o->value.string.string);
} else {
// Don't really know what to do about sort quoting
// for arbitrary objects
@@ -246,8 +247,10 @@ void join_csv(json_object *j) {
}
std::string joinkey;
if (key->type == JSON_STRING || key->type == JSON_NUMBER) {
joinkey = key->string;
if (key->type == JSON_STRING) {
joinkey = key->value.string.string;
} else if (key->type == JSON_NUMBER) {
joinkey = milo::dtoa_milo(key->value.number.number);
} else {
const char *s = json_stringify(key);
joinkey = s;
@@ -302,9 +305,10 @@ void join_csv(json_object *j) {
if (fields.size() > 0 && joinkey == fields[0]) {
// This knows more about the structure of JSON objects than it ought to
properties->keys = (json_object **) realloc((void *) properties->keys, (properties->length + 32 + fields.size()) * sizeof(json_object *));
properties->values = (json_object **) realloc((void *) properties->values, (properties->length + 32 + fields.size()) * sizeof(json_object *));
if (properties->keys == NULL || properties->values == NULL) {
// The 8 is to round up at least as much as SIZE_FOR in json_pull.c
properties->value.object.keys = (json_object **) realloc((void *) properties->value.object.keys, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *));
properties->value.object.values = (json_object **) realloc((void *) properties->value.object.values, (properties->value.object.length + 8 + fields.size()) * sizeof(json_object *));
if (properties->value.object.keys == NULL || properties->value.object.values == NULL) {
perror("realloc");
exit(EXIT_FAILURE);
}
@@ -335,29 +339,36 @@ void join_csv(json_object *j) {
}
ko->type = JSON_STRING;
vo->type = attr_type;
ko->parent = properties;
ko->parser = properties->parser;
ko->parent = vo->parent = properties;
ko->array = vo->array = NULL;
ko->keys = vo->keys = NULL;
ko->values = vo->values = NULL;
ko->parser = vo->parser = properties->parser;
ko->string = strdup(k.c_str());
vo->string = strdup(v.c_str());
if (ko->string == NULL || vo->string == NULL) {
ko->value.string.string = strdup(k.c_str());
if (ko->value.string.string == NULL) {
perror("strdup");
exit(EXIT_FAILURE);
}
ko->length = strlen(ko->string);
vo->length = strlen(vo->string);
vo->number = atof(vo->string);
vo->type = attr_type;
vo->parent = properties;
vo->parser = properties->parser;
properties->keys[properties->length] = ko;
properties->values[properties->length] = vo;
properties->length++;
if (attr_type == JSON_STRING) {
vo->value.string.string = strdup(v.c_str());
if (vo->value.string.string == NULL) {
perror("strdup");
exit(EXIT_FAILURE);
}
} else if (attr_type == JSON_NUMBER) {
vo->value.number.number = atof(v.c_str());
vo->value.number.large_unsigned = 0;
vo->value.number.large_signed = 0;
} else {
abort();
}
properties->value.object.keys[properties->value.object.length] = ko;
properties->value.object.values[properties->value.object.length] = vo;
properties->value.object.length++;
}
}
}