diff --git a/src/gfx/process.cpp b/src/gfx/process.cpp
index 66b6c010..bbd86b02 100644
--- a/src/gfx/process.cpp
+++ b/src/gfx/process.cpp
@@ -218,6 +218,13 @@ struct Image {
 				}
 			}
 		}
+		if (size_t nbAmbiguousColors = ambiguous.size(); nbAmbiguousColors > 0) {
+			fatal(
+			    "Image contains %zu ambiguous color%s (neither transparent nor opaque)",
+			    nbAmbiguousColors,
+			    nbAmbiguousColors == 1 ? "" : "s"
+			);
+		}
 	}
 
 	class TilesVisitor {
@@ -1056,12 +1063,14 @@ void process() {
 	for (auto tile : image.visitAsTiles()) {
 		AttrmapEntry &attrs = attrmap.emplace_back();
 
-		// Count the unique non-transparent colors for packing
+		// Count the unique opaque colors for packing
 		std::unordered_set<uint16_t> tileColors;
 		for (uint32_t y = 0; y < 8; ++y) {
 			for (uint32_t x = 0; x < 8; ++x) {
-				if (Rgba color = tile.pixel(x, y);
-				    color.isOpaque() || !options.hasTransparentPixels) {
+				Rgba color = tile.pixel(x, y);
+				// Ambiguous colors should not be in `tileColors`
+				assume(color.isOpaque() != color.isTransparent());
+				if (color.isOpaque() || !options.hasTransparentPixels) {
 					tileColors.insert(color.cgbColor());
 				}
 			}
diff --git a/test/gfx/ambiguous.err b/test/gfx/ambiguous.err
index d7817a14..3a74e624 100644
--- a/test/gfx/ambiguous.err
+++ b/test/gfx/ambiguous.err
@@ -1,2 +1,3 @@
 error: Color #ff800080 is neither transparent (alpha < 16) nor opaque (alpha >= 240) (first seen at (0, 8))
-Conversion aborted after 1 error
+FATAL: Image contains 1 ambiguous color (neither transparent nor opaque)
+Conversion aborted after 2 errors
diff --git a/test/gfx/fully_ambiguous.err b/test/gfx/fully_ambiguous.err
new file mode 100644
index 00000000..52fd9c76
--- /dev/null
+++ b/test/gfx/fully_ambiguous.err
@@ -0,0 +1,4 @@
+error: Color #ff000080 is neither transparent (alpha < 16) nor opaque (alpha >= 240) (first seen at (0, 0))
+error: Color #0000ff80 is neither transparent (alpha < 16) nor opaque (alpha >= 240) (first seen at (0, 4))
+FATAL: Image contains 2 ambiguous colors (neither transparent nor opaque)
+Conversion aborted after 3 errors
diff --git a/test/gfx/fully_ambiguous.png b/test/gfx/fully_ambiguous.png
new file mode 100644
index 00000000..d5bbd515
Binary files /dev/null and b/test/gfx/fully_ambiguous.png differ