From 3f45b5c51c4f0428d130e9b70fd6999ac1210a44 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Fri, 15 Mar 2024 12:17:32 -0700 Subject: [PATCH] Reduce test diff noisiness with intermediate rounding --- serial.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/serial.cpp b/serial.cpp index 35a7c438..f3d088ec 100644 --- a/serial.cpp +++ b/serial.cpp @@ -370,6 +370,15 @@ static long long scale_geometry(struct serialization_state *sst, long long *bbox geom[i].x = std::round(x * scale); geom[i].y = std::round(y * scale); + } else if (geometry_scale >= GLOBAL_DETAIL - 32) { + // this is silly, but rounding coordinates to what they would have been + // with a 32-bit world extent helps make test expectations not change + + x = std::llround((double) x / (1LL << (GLOBAL_DETAIL - 32))) * (1LL << (GLOBAL_DETAIL - 32)); + y = std::llround((double) y / (1LL << (GLOBAL_DETAIL - 32))) * (1LL << (GLOBAL_DETAIL - 32)); + + geom[i].x = SHIFT_RIGHT(x); + geom[i].y = SHIFT_RIGHT(y); } else { geom[i].x = SHIFT_RIGHT(x); geom[i].y = SHIFT_RIGHT(y);