From d13ba7ca530c6063e3bab626f6d679c999ec933f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Pavel=20L=C3=B6bl?= Date: Sun, 6 May 2012 11:03:17 +0200 Subject: [PATCH] Fix floating precision bug When calculating coordinates we should multiply at first otherwise we lose precision when i3 is compiled without sse2 support. The following code prints "Res1: 348 Res2: 349" when compiled with -O0 -mno-sse2 and "Res1: 349 Res2: 349" with -O0 -msee2. Note that -msse2 is default flag on 64bit OSes. int main() { double a = 349.0 / 768; double b = 349.0 * 768; int res1 = a * 768; int res2 = b / 768; printf("Res1: %d Res2: %d\n", res1, res2); return 0; } Thanks guys for helping me to hunt down this one. --- src/floating.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/floating.c b/src/floating.c index 3b691169..c0154aca 100644 --- a/src/floating.c +++ b/src/floating.c @@ -614,12 +614,12 @@ void floating_fix_coordinates(Con *con, Rect *old_rect, Rect *new_rect) { uint32_t rel_y = (con->rect.y - old_rect->y); /* Then we calculate a fraction, for example 0.63 for a window * which is at y = 1212 of a 1920 px high output */ - double fraction_x = ((double)rel_x / old_rect->width); - double fraction_y = ((double)rel_y / old_rect->height); DLOG("rel_x = %d, rel_y = %d, fraction_x = %f, fraction_y = %f, output->w = %d, output->h = %d\n", - rel_x, rel_y, fraction_x, fraction_y, old_rect->width, old_rect->height); - con->rect.x = new_rect->x + (fraction_x * new_rect->width); - con->rect.y = new_rect->y + (fraction_y * new_rect->height); + rel_x, rel_y, (double)rel_x / old_rect->width, (double)rel_y / old_rect->height, + old_rect->width, old_rect->height); + /* Here we have to multiply at first. Or we will lose precision when not compiled with -msse2 */ + con->rect.x = new_rect->x + (double)(rel_x * new_rect->width) / old_rect->width; + con->rect.y = new_rect->y + (double)(rel_y * new_rect->height) / old_rect->height; DLOG("Resulting coordinates: x = %d, y = %d\n", con->rect.x, con->rect.y); } -- 2.39.5