From bc6b5f3d96851a111dce193475fda0a57d4d7715 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 30 Jan 2025 12:23:42 -0800 Subject: [PATCH] Switch mean over to internal accumulation state --- attribute.cpp | 18 ++++++------------ mvt.hpp | 2 +- serial.hpp | 2 +- ...Etheconcat%3aconcat_-Ethecomma%3acomma.json | 4 ++-- tile.cpp | 5 +++++ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/attribute.cpp b/attribute.cpp index 2feb224d..0886c70f 100644 --- a/attribute.cpp +++ b/attribute.cpp @@ -120,19 +120,13 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key, } case op_mean: { - auto state = attribute_accum_state.find(key); - if (state == attribute_accum_state.end()) { - accum_state s; - s.sum = full_values[i].to_double() + val.to_double(); - s.count = 2; - attribute_accum_state.insert(std::pair(key, s)); - - full_values[i] = (s.sum / s.count); + size_t count = full_values[i].get_count(); + if (count <= 1) { + full_values[i].set_double_count((full_values[i].to_double() + val.to_double()) / 2, 2); } else { - state->second.sum += val.to_double(); - state->second.count += 1; - - full_values[i] = (state->second.sum / state->second.count); + double sum = full_values[i].to_double() * count + val.to_double(); + count++; + full_values[i].set_double_count(sum / count, count); } return; } diff --git a/mvt.hpp b/mvt.hpp index 800021d3..8f24fc68 100644 --- a/mvt.hpp +++ b/mvt.hpp @@ -159,7 +159,7 @@ struct mvt_value { this->count = 0; } - void set_double_accum(double v, size_t c) { + void set_double_count(double v, size_t c) { this->type = mvt_double; this->numeric_value.double_value = v; this->count = c; diff --git a/serial.hpp b/serial.hpp index 0fb9c5c0..8437739f 100644 --- a/serial.hpp +++ b/serial.hpp @@ -80,7 +80,7 @@ struct serial_val { } } - void set_double_accum(double v, size_t c) { + void set_double_count(double v, size_t c) { type = mvt_double; s = milo::dtoa_milo(v) + '\0' + std::to_string(c); } diff --git a/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json b/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json index 9328c90c..a002d18f 100644 --- a/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json +++ b/tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json @@ -5,7 +5,7 @@ "description": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles", "format": "pbf", "generator_options": "./tippecanoe -q -a@ -f -o tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles -z5 -Ethesum:sum -Etheproduct:product -Ethemax:max -Ethemin:min -Ethemean:mean -Etheconcat:concat -Ethecomma:comma tests/accumulate/in.json", -"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.773809523809529,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}", +"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.77380952380951,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}", "maxzoom": "5", "minzoom": "0", "name": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles", @@ -15,7 +15,7 @@ }, "features": [ { "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [ -{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.773809523809529, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } } +{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.77380952380951, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } } ] } ] } , diff --git a/tile.cpp b/tile.cpp index 108da534..ef481cfa 100644 --- a/tile.cpp +++ b/tile.cpp @@ -2370,6 +2370,11 @@ long long write_tile(decompressor *geoms, std::atomic *geompos_in, ch if (p.need_tilestats.size() > 0) { for (size_t j = 0; j < p.full_keys.size(); j++) { if (p.need_tilestats.count(*p.full_keys[j]) > 0) { + // remove accumulation state + size_t found = p.full_values[j].s.find('\0'); + if (found != std::string::npos) { + p.full_values[j].s = p.full_values[j].s.substr(0, found); + } add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]); } }