Compare commits

...
22 Commits
Author SHA1 Message Date
Erica Fischer dda13b3c97 Case smashing and prefix trimming in the select 2025-01-03 10:26:54 -08:00
Erica Fischer b87d8e4560 Fix accidental inclusion of the top left of the tile in the bbox 2024-12-20 10:41:53 -08:00
Erica Fischer 632d031caa Missed a test 2024-12-20 09:46:22 -08:00
Erica Fischer 4788b67a54 Gather the results from each thread *after* the thread finishes 2024-12-20 09:42:09 -08:00
Erica Fischer 7338c4b6b0 An empty tileset has empty bounds at null island 2024-12-19 16:24:01 -08:00
Erica Fischer acffe23b97 More tests 2024-12-19 16:18:58 -08:00
Erica Fischer d6ce92422f Make tile-join bounding boxes reflect feature bounds, not tile bounds 2024-12-19 16:16:52 -08:00
Erica Fischer 5c8f1545a5 Add a flag to exclude all attributes from the tile side of the join 2024-12-17 15:43:34 -08:00
Erica Fischer 638a7617d0 Merge remote-tracking branch 'origin/main' into tile-join-sqlite 2024-12-17 15:23:01 -08:00
Erica Fischer fcc611adc4 Observe attribute exclusion when joining from sql queries 2024-12-17 15:22:34 -08:00
Erica Fischer 03daaefd8e Add a flag to get the feature ID from the query 2024-12-17 15:01:06 -08:00
Erica Fischer c88f8c07e7 Set matched if the sql join matches 2024-12-17 14:43:04 -08:00
Erica Fischer a4b866833a Join the attributes onto the feature 2024-12-05 16:19:54 -08:00
Erica Fischer cbcdbc5516 Actually do the query and get results back 2024-12-05 16:01:27 -08:00
Erica Fischer ad47b93a81 Actually open the gpkg. Fix the query quoting. 2024-12-05 14:42:48 -08:00
Erica Fischer 264665b0ce Make a query 2024-12-05 13:52:28 -08:00
Erica Fischer eab74d4700 Merge remote-tracking branch 'origin/main' into tile-join-sqlite 2024-12-05 12:39:51 -08:00
Erica Fischer 0ae73c1143 Gathering join keys for sql query 2024-12-03 16:16:41 -08:00
Erica Fischer 36b207c16e Fix some unnecessary round trips from std::string to char * and back 2024-12-03 14:43:02 -08:00
Erica Fischer a07aad9db7 Enable sqlite3 serialized multithreading 2024-12-02 14:21:53 -08:00
Erica Fischer f2d149059e Merge remote-tracking branch 'origin/main' into tile-join-sqlite 2024-12-02 11:35:28 -08:00
Erica Fischer 5c57d5aef2 Sketching out sqlite options for tile-join 2024-11-04 12:17:25 -08:00
21 changed files with 321 additions and 143 deletions
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129",
"bounds": "-180.000000,-85.051129,180.000000,85.051129",
"antimeridian_adjusted_bounds": "-100.019531,0.000000,0.966797,0.966751",
"bounds": "-100.019531,0.000000,0.966797,0.966751",
"center": "0.000000,0.000000,0",
"description": "tests/feature-filter/out/all.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-175.781250,-42.032974,180.000000,64.168107",
"bounds": "-175.781250,-42.032974,180.000000,64.168107",
"antimeridian_adjusted_bounds": "-175.220947,-41.302571,179.217224,64.168107",
"bounds": "-175.220947,-41.302571,179.217224,64.168107",
"center": "-62.578125,17.307462,8",
"description": "tests/feature-filter/out/places.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-180.000000,-85.051129,180.000000,85.051129",
"bounds": "-180.000000,-85.051129,180.000000,85.051129",
"antimeridian_adjusted_bounds": "-122.255859,37.718590,-122.080078,37.857507",
"bounds": "-122.255859,37.718590,-122.080078,37.857507",
"center": "-122.104097,37.695438,0",
"description": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.309418,37.881357,-122.280579,37.900865",
"bounds": "-122.309418,37.881357,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.857507,-122.255859,37.926868",
"bounds": "-122.343750,37.857507,-122.255859,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,7 +1,7 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649",
"attribution": "macarthur's attribution",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"bounds": "-122.294655,37.694688,-122.103424,37.833649",
"center": "-122.299805,37.892187,11",
"description": "macarthur description",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107",
"bounds": "-122.294655,37.695231,-122.103424,37.833107",
"center": "-122.167969,37.833010,9",
"description": "tests/join-population/macarthur.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
"antimeridian_adjusted_bounds": "-122.294655,37.695231,-122.103424,37.833107",
"bounds": "-122.294655,37.695231,-122.103424,37.833107",
"center": "-122.167969,37.833010,9",
"description": "tests/join-population/macarthur.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865",
"bounds": "-122.343750,37.694688,-122.103424,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420-folder",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865",
"bounds": "-122.343750,37.694688,-122.103424,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.874853,-122.280579,37.900865",
"bounds": "-122.343750,37.874853,-122.280579,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "237.656250,37.857507,237.744141,37.926868",
"bounds": "-122.343750,37.695438,-122.104097,37.926868",
"antimeridian_adjusted_bounds": "-122.343750,37.694688,-122.103424,37.900865",
"bounds": "-122.343750,37.694688,-122.103424,37.900865",
"center": "-122.299805,37.892187,12",
"description": "tests/join-population/tabblock_06001420.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649",
"bounds": "-122.294655,37.694688,-122.103424,37.833649",
"center": "-122.167969,37.828608,10",
"description": "tests/join-population/macarthur2.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-122.343750,37.439974,-121.992188,37.996163",
"bounds": "-122.343750,37.439974,-121.992188,37.996163",
"antimeridian_adjusted_bounds": "-122.294655,37.694688,-122.103424,37.833649",
"bounds": "-122.294655,37.694688,-122.103424,37.833649",
"center": "-122.167969,37.833010,10",
"description": "tests/join-population/macarthur.mbtiles",
"format": "pbf",
+2 -2
View File
@@ -1,6 +1,6 @@
{ "type": "FeatureCollection", "properties": {
"antimeridian_adjusted_bounds": "-123.750000,45.089036,-120.937500,47.040182",
"bounds": "-123.750000,45.089036,-120.937500,47.040182",
"antimeridian_adjusted_bounds": "-122.682953,45.512121,-122.654800,45.569832",
"bounds": "-122.682953,45.512121,-122.654800,45.569832",
"center": "-122.662354,45.514045,7",
"description": "tests/raw-tiles/raw-tiles",
"format": "pbf",
+281 -103
View File
@@ -55,7 +55,12 @@ int maxzoom = 32;
int minzoom = 0;
std::map<std::string, std::string> renames;
bool exclude_all = false;
bool exclude_all_tile_attributes = false;
std::vector<std::string> unidecode_data;
std::string join_tile_column;
std::string join_table_column;
std::string join_table;
std::string attribute_for_id;
bool want_overzoom = false;
int buffer = 5;
@@ -73,7 +78,108 @@ struct stats {
std::vector<struct strategy> strategies{};
};
void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &include, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter) {
std::vector<std::map<std::string, mvt_value>> get_joined_rows(sqlite3 *db, const std::vector<mvt_value> &join_keys) {
std::vector<std::map<std::string, mvt_value>> ret;
ret.resize(join_keys.size());
// double quotes for table and column identifiers
const char *s = sqlite3_mprintf("select LOWER(LTRIM(SUBSTR(\"%w\",1,LENGTH(\"%w\")-3),'0')||SUBSTR(\"%w\",-3,3)), * from \"%w\" where LOWER(LTRIM(SUBSTR(\"%w\",1,LENGTH(\"%w\")-3),'0')||SUBSTR(\"%w\",-3,3)) in (",
join_table_column.c_str(), join_table_column.c_str(), join_table_column.c_str(),
join_table.c_str(),
join_table_column.c_str(), join_table_column.c_str(), join_table_column.c_str());
std::string query = s;
sqlite3_free((void *) s);
std::map<std::string, size_t> key_to_row;
for (size_t i = 0; i < join_keys.size(); i++) {
const mvt_value &v = join_keys[i];
// single quotes for literals
if (v.type == mvt_string) {
s = sqlite3_mprintf("'%q'", v.c_str());
query += s;
sqlite3_free((void *) s);
key_to_row.emplace(v.get_string_value(), i);
} else {
std::string stringified = v.toString();
key_to_row.emplace(stringified, i);
query += stringified;
}
if (i + 1 < join_keys.size()) {
query += ", ";
}
}
query += ");";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, query.c_str(), -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "sqlite3 query %s failed: %s\n", query.c_str(), sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
while (sqlite3_step(stmt) == SQLITE_ROW) {
int count = sqlite3_column_count(stmt);
std::map<std::string, mvt_value> row;
if (count > 0) {
// join key is 0th column of query
std::string key = (const char *) sqlite3_column_text(stmt, 0);
auto f = key_to_row.find(key);
if (f == key_to_row.end()) {
fprintf(stderr, "Unexpected join key: %s\n", key.c_str());
continue;
}
for (int i = 1; i < count; i++) {
int type = sqlite3_column_type(stmt, i);
mvt_value v;
v.type = mvt_null;
if (type == SQLITE_INTEGER || type == SQLITE_FLOAT) {
v = mvt_value(sqlite3_column_double(stmt, i));
} else if (type == SQLITE_TEXT || type == SQLITE_BLOB) {
v.set_string_value((const char *) sqlite3_column_text(stmt, i));
}
const char *name = sqlite3_column_name(stmt, i);
row.emplace(name, v);
}
ret[f->second] = std::move(row);
}
}
if (sqlite3_finalize(stmt) != SQLITE_OK) {
fprintf(stderr, "sqlite3 finalize failed: %s\n", sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
return ret;
}
struct arg {
std::map<zxy, std::vector<std::string>> inputs{};
std::map<zxy, std::string> outputs{};
std::map<std::string, layermap_entry> *layermap = NULL;
std::vector<std::string> *header = NULL;
std::map<std::string, std::vector<std::string>> *mapping = NULL;
sqlite3 *db = NULL;
std::set<std::string> *exclude = NULL;
std::set<std::string> *include = NULL;
std::set<std::string> *keep_layers = NULL;
std::set<std::string> *remove_layers = NULL;
int ifmatched = 0;
json_object *filter = NULL;
struct tileset_reader *readers = NULL;
double minlat, minlon;
double maxlat, maxlon;
double minlon2, maxlon2;
};
void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<std::string, layermap_entry> &layermap, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, sqlite3 *db, std::set<std::string> &exclude, std::set<std::string> &include, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, int ifmatched, mvt_tile &outtile, json_object *filter, struct arg *a) {
mvt_tile tile;
int features_added = 0;
bool was_compressed;
@@ -139,12 +245,42 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
}
}
std::vector<std::map<std::string, mvt_value>> joined;
if (db != NULL) {
// collect join keys for sql query
std::vector<mvt_value> join_keys;
join_keys.resize(layer.features.size());
for (size_t f = 0; f < layer.features.size(); f++) {
mvt_feature &feat = layer.features[f];
join_keys[f].type = mvt_no_such_key;
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
const std::string &key = layer.keys[feat.tags[t]];
if (key == join_tile_column) {
const mvt_value &val = layer.values[feat.tags[t + 1]];
join_keys[f] = val;
break;
}
}
}
joined = get_joined_rows(db, join_keys);
}
auto tilestats = layermap.find(layer.name);
for (size_t f = 0; f < layer.features.size(); f++) {
mvt_feature feat = layer.features[f];
std::set<std::string> exclude_attributes;
long long minx = LLONG_MAX;
long long miny = LLONG_MAX;
long long maxx = LLONG_MIN;
long long maxy = LLONG_MIN;
bool features_added_to_layer = false;
for (size_t f = 0; f < layer.features.size(); f++) {
mvt_feature &feat = layer.features[f];
std::set<std::string> exclude_attributes;
if (filter != NULL && !evaluate(feat, layer, filter, exclude_attributes, z, unidecode_data)) {
continue;
}
@@ -161,7 +297,7 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
std::vector<std::string> key_order;
for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) {
const char *key = layer.keys[feat.tags[t]].c_str();
const std::string &key = layer.keys[feat.tags[t]];
mvt_value &val = layer.values[feat.tags[t + 1]];
serial_val sv = mvt_value_to_serial_val(val);
@@ -169,12 +305,32 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
continue;
}
if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) {
attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(key, std::pair<mvt_value, serial_val>(val, sv)));
key_order.push_back(key);
if (!exclude_all_tile_attributes) {
if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) {
attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(key, std::pair<mvt_value, serial_val>(val, sv)));
key_order.push_back(key);
}
}
if (header.size() > 0 && strcmp(key, header[0].c_str()) == 0) {
if (f < joined.size()) {
if (joined[f].size() > 0) {
matched = true;
}
for (auto const &kv : joined[f]) {
if (kv.first == attribute_for_id) {
outfeature.has_id = true;
outfeature.id = mvt_value_to_long_long(kv.second);
} else if (include.count(kv.first) || (!exclude_all && exclude.count(kv.first) == 0 && exclude_attributes.count(kv.first) == 0)) {
if (kv.second.type != mvt_null) {
attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(kv.first, std::pair<mvt_value, serial_val>(kv.second, mvt_value_to_serial_val(kv.second))));
key_order.push_back(kv.first);
}
}
}
}
if (header.size() > 0 && key == header[0]) {
std::map<std::string, std::vector<std::string>>::iterator ii = mapping.find(sv.s);
if (ii != mapping.end()) {
@@ -217,7 +373,6 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
outsv.type = outval.type;
outsv.s = joinval;
// Convert from double to int if the joined attribute is an integer
outval = stringified_to_mvt_value(outval.type, joinval.c_str(), tile_stringpool);
attributes.insert(std::pair<std::string, std::pair<mvt_value, serial_val>>(joinkey, std::pair<mvt_value, serial_val>(outval, outsv)));
@@ -257,7 +412,22 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
}
}
for (auto const &g : outfeature.geometry) {
if (g.op == mvt_moveto || g.op == mvt_lineto) {
// pin to the tile extent, since we don't want bounds bigger than the earth
long long gx = std::min((long long) outlayer.extent, std::max(0LL, g.x));
long long gy = std::min((long long) outlayer.extent, std::max(0LL, g.y));
// initially keep bounds in tile coordinates
minx = std::min(minx, gx);
miny = std::min(miny, gy);
maxx = std::max(maxx, gx);
maxy = std::max(maxy, gy);
}
}
features_added++;
features_added_to_layer = true;
outlayer.features.push_back(outfeature);
if (z < tilestats->second.minzoom) {
@@ -276,6 +446,38 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
}
}
}
if (features_added_to_layer) {
// to world scale
minx = minx * (1LL << (32 - z)) / outlayer.extent;
miny = miny * (1LL << (32 - z)) / outlayer.extent;
maxx = maxx * (1LL << (32 - z)) / outlayer.extent;
maxy = maxy * (1LL << (32 - z)) / outlayer.extent;
// to world offset
minx += (1LL << (32 - z)) * x;
maxx += (1LL << (32 - z)) * x;
miny += (1LL << (32 - z)) * y;
maxy += (1LL << (32 - z)) * y;
double lat1, lon1;
double lat2, lon2;
tile2lonlat(minx, maxy, 32, &lon1, &lat1);
tile2lonlat(maxx, miny, 32, &lon2, &lat2);
a->minlat = std::min(a->minlat, std::min(lat1, lat2));
a->minlon = std::min(a->minlon, std::min(lon1, lon2));
a->maxlat = std::max(a->maxlat, std::max(lat1, lat2));
a->maxlon = std::max(a->maxlon, std::max(lon1, lon2));
if (lon1 < 0 || lon2 < 0) {
lon1 += 360;
lon2 += 360;
}
a->minlon2 = std::min(a->minlon2, std::min(lon1, lon2));
a->maxlon2 = std::max(a->maxlon2, std::max(lon1, lon2));
}
}
if (features_added == 0) {
@@ -283,22 +485,6 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map<st
}
}
double min(double a, double b) {
if (a < b) {
return a;
} else {
return b;
}
}
double max(double a, double b) {
if (a > b) {
return a;
} else {
return b;
}
}
struct tilecmp {
bool operator()(std::pair<unsigned, unsigned> const &a, std::pair<unsigned, unsigned> const &b) {
// must match behavior of tileset_reader::operator<()
@@ -736,23 +922,6 @@ struct tileset_reader *begin_reading(char *fname) {
return r;
}
struct arg {
std::map<zxy, std::vector<std::string>> inputs{};
std::map<zxy, std::string> outputs{};
std::map<std::string, layermap_entry> *layermap = NULL;
std::vector<std::string> *header = NULL;
std::map<std::string, std::vector<std::string>> *mapping = NULL;
std::set<std::string> *exclude = NULL;
std::set<std::string> *include = NULL;
std::set<std::string> *keep_layers = NULL;
std::set<std::string> *remove_layers = NULL;
int ifmatched = 0;
json_object *filter = NULL;
struct tileset_reader *readers = NULL;
};
void *join_worker(void *v) {
arg *a = (arg *) v;
@@ -760,7 +929,7 @@ void *join_worker(void *v) {
mvt_tile tile;
for (size_t i = 0; i < ai->second.size(); i++) {
append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter);
append_tile(ai->second[i], ai->first.z, ai->first.x, ai->first.y, *(a->layermap), *(a->header), *(a->mapping), a->db, *(a->exclude), *(a->include), *(a->keep_layers), *(a->remove_layers), a->ifmatched, tile, a->filter, a);
}
ai->second.clear();
@@ -795,7 +964,7 @@ void *join_worker(void *v) {
return NULL;
}
void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &include, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, json_object *filter, struct tileset_reader *readers) {
void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<std::map<std::string, layermap_entry>> &layermaps, sqlite3 *outdb, const char *outdir, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, sqlite3 *db, std::set<std::string> &exclude, std::set<std::string> &include, int ifmatched, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, json_object *filter, struct tileset_reader *readers, double *minlat, double *minlon, double *maxlat, double *maxlon, double *minlon2, double *maxlon2) {
pthread_t pthreads[CPUS];
std::vector<arg> args;
@@ -805,6 +974,7 @@ void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<
args[i].layermap = &layermaps[i];
args[i].header = &header;
args[i].mapping = &mapping;
args[i].db = db;
args[i].exclude = &exclude;
args[i].include = &include;
args[i].keep_layers = &keep_layers;
@@ -812,6 +982,12 @@ void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<
args[i].ifmatched = ifmatched;
args[i].filter = filter;
args[i].readers = readers;
args[i].minlat = *minlat;
args[i].minlon = *minlon;
args[i].maxlat = *maxlat;
args[i].maxlon = *maxlon;
args[i].minlon2 = *minlon2;
args[i].maxlon2 = *maxlon2;
}
size_t count = 0;
@@ -844,6 +1020,13 @@ void dispatch_tasks(std::map<zxy, std::vector<std::string>> &tasks, std::vector<
perror("pthread_join");
}
*minlat = std::min(*minlat, args[i].minlat);
*minlon = std::min(*minlon, args[i].minlon);
*maxlat = std::max(*maxlat, args[i].maxlat);
*maxlon = std::max(*maxlon, args[i].maxlon);
*minlon2 = std::min(*minlon2, args[i].minlon2);
*maxlon2 = std::max(*maxlon2, args[i].maxlon2);
for (auto ai = args[i].outputs.begin(); ai != args[i].outputs.end(); ++ai) {
if (outdb != NULL) {
mbtiles_write_tile(outdb, ai->first.z, ai->first.x, ai->first.y, ai->second.data(), ai->second.size());
@@ -945,7 +1128,7 @@ void handle_vector_layers(json_object *vector_layers, std::map<std::string, laye
}
}
void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, std::set<std::string> &exclude, std::set<std::string> &include, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name, json_object *filter, std::map<std::string, std::string> &attribute_descriptions, std::string &generator_options, std::vector<strategy> *strategies) {
void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry> &layermap, sqlite3 *outdb, const char *outdir, struct stats *st, std::vector<std::string> &header, std::map<std::string, std::vector<std::string>> &mapping, sqlite3 *db, std::set<std::string> &exclude, std::set<std::string> &include, int ifmatched, std::string &attribution, std::string &description, std::set<std::string> &keep_layers, std::set<std::string> &remove_layers, std::string &name, json_object *filter, std::map<std::string, std::string> &attribute_descriptions, std::string &generator_options, std::vector<strategy> *strategies) {
std::vector<std::map<std::string, layermap_entry>> layermaps;
for (size_t i = 0; i < CPUS; i++) {
layermaps.push_back(std::map<std::string, layermap_entry>());
@@ -958,36 +1141,10 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
double maxlon = INT_MIN;
double minlon2 = INT_MAX;
double maxlon2 = INT_MIN;
int zoom_for_bbox = -1;
while (readers != NULL && !readers->all_done()) {
std::pair<zxy, std::string> current = readers->current();
if (current.first.z != zoom_for_bbox) {
// Only use highest zoom for bbox calculation
// to avoid z0 always covering the world
minlat = minlon = minlon2 = INT_MAX;
maxlat = maxlon = maxlon2 = INT_MIN;
zoom_for_bbox = current.first.z;
}
double lat1, lon1, lat2, lon2;
tile2lonlat(current.first.x, current.first.y, current.first.z, &lon1, &lat1);
tile2lonlat(current.first.x + 1, current.first.y + 1, current.first.z, &lon2, &lat2);
minlat = min(lat2, minlat);
minlon = min(lon1, minlon);
maxlat = max(lat1, maxlat);
maxlon = max(lon2, maxlon);
if (lon1 < 0) {
lon1 += 360;
lon2 += 360;
}
minlon2 = min(lon1, minlon2);
maxlon2 = max(lon2, maxlon2);
if (current.first.z >= minzoom && current.first.z <= maxzoom) {
zxy tile = current.first;
if (tasks.count(tile) == 0) {
@@ -1014,7 +1171,7 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
if (readers == NULL || readers->zoom != current.first.z || readers->x != current.first.x || readers->y != current.first.y) {
if (tasks.size() > 100 * CPUS) {
dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers);
dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2);
tasks.clear();
}
}
@@ -1033,19 +1190,19 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
*rr = r;
}
st->minlon = min(minlon, st->minlon);
st->maxlon = max(maxlon, st->maxlon);
st->minlat = min(minlat, st->minlat);
st->maxlat = max(maxlat, st->maxlat);
st->minlon2 = min(minlon2, st->minlon2);
st->maxlon2 = max(maxlon2, st->maxlon2);
st->minlat2 = min(minlat, st->minlat2);
st->maxlat2 = max(maxlat, st->maxlat2);
dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers);
dispatch_tasks(tasks, layermaps, outdb, outdir, header, mapping, db, exclude, include, ifmatched, keep_layers, remove_layers, filter, readers, &minlat, &minlon, &maxlat, &maxlon, &minlon2, &maxlon2);
layermap = merge_layermaps(layermaps);
st->minlon = std::min(minlon, st->minlon);
st->maxlon = std::max(maxlon, st->maxlon);
st->minlat = std::min(minlat, st->minlat);
st->maxlat = std::max(maxlat, st->maxlat);
st->minlon2 = std::min(minlon2, st->minlon2);
st->maxlon2 = std::max(maxlon2, st->maxlon2);
st->minlat2 = std::min(minlat, st->minlat2);
st->maxlat2 = std::max(maxlat, st->maxlat2);
struct tileset_reader *next;
for (struct tileset_reader *r = readers; r != NULL; r = next) {
next = r->next;
@@ -1054,14 +1211,14 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'minzoom'", -1, &stmt, NULL) == SQLITE_OK) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
int minz = max(sqlite3_column_int(stmt, 0), minzoom);
st->minzoom = min(st->minzoom, minz);
int minz = std::max(sqlite3_column_int(stmt, 0), minzoom);
st->minzoom = std::min(st->minzoom, minz);
}
sqlite3_finalize(stmt);
}
if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'maxzoom'", -1, &stmt, NULL) == SQLITE_OK) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
int maxz = min(sqlite3_column_int(stmt, 0), maxzoom);
int maxz = std::min(sqlite3_column_int(stmt, 0), maxzoom);
if (!want_overzoom) {
if (st->maxzoom >= 0 && maxz != st->maxzoom) {
@@ -1069,7 +1226,7 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
}
}
st->maxzoom = max(st->maxzoom, maxz);
st->maxzoom = std::max(st->maxzoom, maxz);
}
sqlite3_finalize(stmt);
}
@@ -1116,20 +1273,6 @@ void decode(struct tileset_reader *readers, std::map<std::string, layermap_entry
}
sqlite3_finalize(stmt);
}
if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'bounds'", -1, &stmt, NULL) == SQLITE_OK) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
const unsigned char *s = sqlite3_column_text(stmt, 0);
if (s != NULL) {
if (sscanf((char *) s, "%lf,%lf,%lf,%lf", &minlon, &minlat, &maxlon, &maxlat) == 4) {
st->minlon = min(minlon, st->minlon);
st->maxlon = max(maxlon, st->maxlon);
st->minlat = min(minlat, st->minlat);
st->maxlat = max(maxlat, st->maxlat);
}
}
}
sqlite3_finalize(stmt);
}
if (sqlite3_prepare_v2(r->db, "SELECT value from metadata where name = 'json'", -1, &stmt, NULL) == SQLITE_OK) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
const unsigned char *s = sqlite3_column_text(stmt, 0);
@@ -1198,6 +1341,8 @@ int main(int argc, char **argv) {
int filearg = 0;
json_object *filter = NULL;
std::string join_sqlite_fname;
struct tileset_reader *readers = NULL;
CPUS = get_num_avail_cpus();
@@ -1210,8 +1355,14 @@ int main(int argc, char **argv) {
CPUS = 1;
}
if (sqlite3_config(SQLITE_CONFIG_SERIALIZED) != SQLITE_OK) {
fprintf(stderr, "Could not enable sqlite3 serialized multithreading\n");
exit(EXIT_SQLITE);
}
std::vector<std::string> header;
std::map<std::string, std::vector<std::string>> mapping;
sqlite3 *db = NULL;
std::set<std::string> exclude;
std::set<std::string> include;
@@ -1235,6 +1386,7 @@ int main(int argc, char **argv) {
{"exclude", required_argument, 0, 'x'},
{"exclude-all", no_argument, 0, 'X'},
{"include", required_argument, 0, 'y'},
{"exclude-all-tile-attributes", no_argument, 0, '~'},
{"layer", required_argument, 0, 'l'},
{"exclude-layer", required_argument, 0, 'L'},
{"quiet", no_argument, 0, 'q'},
@@ -1245,6 +1397,12 @@ int main(int argc, char **argv) {
{"rename-layer", required_argument, 0, 'R'},
{"read-from", required_argument, 0, 'r'},
{"join-sqlite", required_argument, 0, '~'},
{"join-tile-column", required_argument, 0, '~'},
{"join-table-column", required_argument, 0, '~'},
{"join-table", required_argument, 0, '~'},
{"use-attribute-for-id", required_argument, 0, '~'},
{"no-tile-size-limit", no_argument, &pk, 1},
{"no-tile-compression", no_argument, &pC, 1},
{"empty-csv-columns-are-null", no_argument, &pe, 1},
@@ -1429,6 +1587,22 @@ int main(int argc, char **argv) {
max_tilestats_values = atoi(optarg);
} else if (strcmp(opt, "unidecode-data") == 0) {
unidecode_data = read_unidecode(optarg);
} else if (strcmp(opt, "join-sqlite") == 0) {
join_sqlite_fname = optarg;
if (sqlite3_open(optarg, &db) != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", optarg, sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
} else if (strcmp(opt, "join-table") == 0) {
join_table = optarg;
} else if (strcmp(opt, "join-table-column") == 0) {
join_table_column = optarg;
} else if (strcmp(opt, "join-tile-column") == 0) {
join_tile_column = optarg;
} else if (strcmp(opt, "use-attribute-for-id") == 0) {
attribute_for_id = optarg;
} else if (strcmp(opt, "exclude-all-tile-attributes") == 0) {
exclude_all_tile_attributes = true;
} else {
fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
exit(EXIT_ARGS);
@@ -1510,7 +1684,7 @@ int main(int argc, char **argv) {
std::string generator_options;
std::vector<strategy> strategies;
decode(readers, layermap, outdb, out_dir, &st, header, mapping, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies);
decode(readers, layermap, outdb, out_dir, &st, header, mapping, db, exclude, include, ifmatched, attribution, description, keep_layers, remove_layers, name, filter, attribute_descriptions, generator_options, &strategies);
if (set_attribution.size() != 0) {
attribution = set_attribution;
@@ -1541,6 +1715,10 @@ int main(int argc, char **argv) {
}
}
if (st.maxlon < st.minlon) {
st.maxlon = st.minlon = st.maxlat = st.minlat = st.minlon2 = st.maxlon2 = st.minlat2 = st.maxlat2 = 0;
}
if (st.maxlon - st.minlon <= st.maxlon2 - st.minlon2) {
st.minlon2 = st.minlon;
st.maxlon2 = st.maxlon;