Fix two bugs with RGBASM fixed-point math (#1388)

- Fixed-point formulas are implemented using IEEE-754 floating-point
  internally, which could give infinity or NaN values whose conversion
  to fixed-point integer was platform-dependent.
- Formatting fixed-point $8000_0000 (INT32_MIN, -2147483648) was
  not putting the negative sign in front.

src/asm/fixpoint.cpp

@@ -25,6 +25,10 @@ static double fix2double(int32_t i, int32_t q) {
 }
 
 static int32_t double2fix(double d, int32_t q) {
+	if (isnan(d))
+		return 0;
+	if (isinf(d))
+		return d < 0 ? INT32_MIN : INT32_MAX;
 	return (int32_t)round(d * pow(2.0, q));
 }