Fix two bugs with RGBASM fixed-point math (#1388)

- Fixed-point formulas are implemented using IEEE-754 floating-point
  internally, which could give infinity or NaN values whose conversion
  to fixed-point integer was platform-dependent.
- Formatting fixed-point $8000_0000 (INT32_MIN, -2147483648) was
  not putting the negative sign in front.

src/asm/fixpoint.cpp

@@ -25,6 +25,10 @@ static double fix2double(int32_t i, int32_t q) {
 }
 
 static int32_t double2fix(double d, int32_t q) {
+	if (isnan(d))
+		return 0;
+	if (isinf(d))
+		return d < 0 ? INT32_MIN : INT32_MAX;
 	return (int32_t)round(d * pow(2.0, q));
 }
 

src/asm/format.cpp

@@ -4,6 +4,7 @@
 
 #include <algorithm>
 #include <inttypes.h>
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -153,11 +154,10 @@ void FormatSpec::appendNumber(std::string &str, uint32_t value) const {
 	char signChar = sign; // 0 or ' ' or '+'
 
 	if (useType == 'd' || useType == 'f') {
-		int32_t v = value;
+		if (int32_t v = value; v < 0) {
-
-		if (v < 0 && v != INT32_MIN) {
 			signChar = '-';
-			value = -v;
+			if (v != INT32_MIN)
+				value = -v;
 		}
 	}
 
@@ -194,16 +194,20 @@ void FormatSpec::appendNumber(std::string &str, uint32_t value) const {
 			useFracWidth = 255;
 		}
 
-		snprintf(
+		double fval = fabs(value / fix_PrecisionFactor());
-		    valueBuf, sizeof(valueBuf), "%.*f", (int)useFracWidth, value / fix_PrecisionFactor()
+		snprintf(valueBuf, sizeof(valueBuf), "%.*f", (int)useFracWidth, fval);
-		);
+	} else if (useType == 'd') {
+		// Decimal numbers may be formatted with a '-' sign by `snprintf`, so `abs` prevents that,
+		// with a special case for `INT32_MIN` since `labs(INT32_MIN)` is UB. The sign will be
+		// printed later from `signChar`.
+		uint32_t uval = value != (uint32_t)INT32_MIN ? labs((int32_t)value) : value;
+		snprintf(valueBuf, sizeof(valueBuf), "%" PRIu32, uval);
 	} else {
-		char const *spec = useType == 'd'   ? "%" PRId32
+		char const *spec = useType == 'u' ? "%" PRIu32
-		                   : useType == 'u' ? "%" PRIu32
 		                   : useType == 'X' ? "%" PRIX32
 		                   : useType == 'x' ? "%" PRIx32
 		                   : useType == 'o' ? "%" PRIo32
-		                                    : "%" PRId32;
+		                                    : "%" PRIu32;
 
 		snprintf(valueBuf, sizeof(valueBuf), spec, value);
 	}

test/asm/format-extremes.asm

@@ -0,0 +1,8 @@
+MACRO test
+	def v = \1
+	println "{#09x:v} = {#012o:v} = {#033b:v} = {u:v}U = {+d:v} = {+.16f:v}"
+ENDM
+	test $7fff_ffff ; INT32_MAX
+	test $8000_0000 ; INT32_MIN
+	test $0000_0000 ; UINT32_MIN
+	test $ffff_ffff ; UINT32_MAX

test/asm/format-extremes.out

@@ -0,0 +1,4 @@
+$7fffffff = &17777777777 = %01111111111111111111111111111111 = 2147483647U = +2147483647 = +32767.9999847412109375
+$80000000 = &20000000000 = %10000000000000000000000000000000 = 2147483648U = -2147483648 = -32768.0000000000000000
+$00000000 = &00000000000 = %00000000000000000000000000000000 = 0U = +0 = +0.0000000000000000
+$ffffffff = &37777777777 = %11111111111111111111111111111111 = 4294967295U = -1 = -0.0000152587890625

test/asm/math.asm

@@ -19,14 +19,18 @@ ENDM
 
 	assert DIV(5.0, 2.0) == 2.5
 	assert DIV(-5.0, 2.0) == -2.5
-	assert DIV(-5.0, 0.0) == $8000_0000
+	assert DIV(5.0, 0.0) == $7fff_ffff ; +inf => INT32_MAX
+	assert DIV(-5.0, 0.0) == $8000_0000 ; -inf => INT32_MIN
+	assert DIV(0.0, 0.0) == $0000_0000 ; nan => 0
 
 	assert MUL(10.0, 0.5) == 5.0
 	assert MUL(10.0, 0.0) == 0.0
 
 	assert FMOD(5.0, 2.0) == 1.0
 	assert FMOD(-5.0, 2.0) == -1.0
-	assert FMOD(-5.0, 0.0) == $8000_0000
+	assert FMOD(5.0, 0.0) == 0 ; nan
+	assert FMOD(-5.0, 0.0) == 0 ; nan
+	assert FMOD(0.0, 0.0) == 0 ; nan
 
 	assert POW(10.0, 2.0) == 100.0
 	assert POW(100.0, 0.5) == 10.0