mirror of
https://github.com/gbdev/rgbds.git
synced 2026-09-16 02:37:06 +00:00
Fix rounding of fixed-point constants where fractional digits should carry into integer part (#2088)
This commit is contained in:
+11
-5
@@ -1005,14 +1005,20 @@ static uint32_t finishReadingFixedPoint(uint32_t integer) {
|
|||||||
precision = options.fixPrecision;
|
precision = options.fixPrecision;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (integer >= (1ULL << (32 - precision))) {
|
|
||||||
warning(WARNING_LARGE_CONSTANT, "Magnitude of fixed-point constant is too large");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cast to unsigned avoids undefined overflow behavior
|
// Cast to unsigned avoids undefined overflow behavior
|
||||||
uint32_t fractional =
|
uint32_t fractional =
|
||||||
static_cast<uint32_t>(round(static_cast<double>(dividend) / divisor * (1ULL << precision)));
|
static_cast<uint32_t>(round(static_cast<double>(dividend) / divisor * (1ULL << precision)));
|
||||||
|
// Carry from `fractional` to `integer` if `round` rounded up to the next integer
|
||||||
|
assume(fractional <= 1ULL << precision);
|
||||||
|
if (fractional == 1ULL << precision) {
|
||||||
|
++integer;
|
||||||
|
fractional = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (integer >= 1ULL << (32 - precision)) {
|
||||||
|
warning(WARNING_LARGE_CONSTANT, "Magnitude of fixed-point constant is too large");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
return (integer << precision) | fractional;
|
return (integer << precision) | fractional;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
opt Q8
|
||||||
|
println strfmt("%f", 1.999)
|
||||||
|
println strfmt("%f", 2.999)
|
||||||
|
println strfmt("%f", 3.999)
|
||||||
|
println strfmt("%f", 16777215.999)
|
||||||
|
|
||||||
|
opt Q16
|
||||||
|
println strfmt("%f", 1.999999)
|
||||||
|
println strfmt("%f", 2.999999)
|
||||||
|
println strfmt("%f", 3.999999)
|
||||||
|
println strfmt("%f", 65535.999999)
|
||||||
|
|
||||||
|
opt Q24
|
||||||
|
println strfmt("%f", 1.999999999)
|
||||||
|
println strfmt("%f", 2.999999999)
|
||||||
|
println strfmt("%f", 3.999999999)
|
||||||
|
println strfmt("%f", 255.999999999)
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
warning: Magnitude of fixed-point constant is too large [-Wlarge-constant]
|
||||||
|
at fixed-point-carry.asm(5)
|
||||||
|
warning: Magnitude of fixed-point constant is too large [-Wlarge-constant]
|
||||||
|
at fixed-point-carry.asm(11)
|
||||||
|
warning: Magnitude of fixed-point constant is too large [-Wlarge-constant]
|
||||||
|
at fixed-point-carry.asm(17)
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
2.00000
|
||||||
|
3.00000
|
||||||
|
4.00000
|
||||||
|
0.00000
|
||||||
|
2.00000
|
||||||
|
3.00000
|
||||||
|
4.00000
|
||||||
|
0.00000
|
||||||
|
2.00000
|
||||||
|
3.00000
|
||||||
|
4.00000
|
||||||
|
0.00000
|
||||||
@@ -5,9 +5,9 @@ $18B
|
|||||||
$62B85
|
$62B85
|
||||||
$4
|
$4
|
||||||
$10000
|
$10000
|
||||||
|
$20000
|
||||||
$10000
|
$10000
|
||||||
$10000
|
$20000
|
||||||
$10000
|
|
||||||
$C570A
|
$C570A
|
||||||
$13333
|
$13333
|
||||||
$13333
|
$13333
|
||||||
|
|||||||
Reference in New Issue
Block a user