Fix detection of tiles with too many colors (#1546)

include/gfx/proto_palette.hpp

@@ -18,12 +18,8 @@ private:
 	std::array<uint16_t, capacity> _colorIndices{UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX};
 
 public:
-	/*
+	// Adds the specified color to the set, or **silently drops it** if the set is full.
-	 * Adds the specified color to the set, or **silently drops it** if the set is full.
+	void add(uint16_t color);
-	 *
-	 * Returns whether the color was unique.
-	 */
-	bool add(uint16_t color);
 
 	enum ComparisonResult {
 		NEITHER,

src/gfx/process.cpp

@@ -1115,33 +1115,44 @@ void process() {
 	DefaultInitVec<AttrmapEntry> attrmap{};
 
 	for (auto tile : png.visitAsTiles()) {
-		ProtoPalette tileColors;
 		AttrmapEntry &attrs = attrmap.emplace_back();
-		uint8_t nbColorsInTile = 0;
 
+		// Count the unique non-transparent colors for packing
+		std::unordered_set<uint16_t> tileColors;
 		for (uint32_t y = 0; y < 8; ++y) {
 			for (uint32_t x = 0; x < 8; ++x) {
-				Rgba color = tile.pixel(x, y);
+				if (Rgba color = tile.pixel(x, y); !color.isTransparent()) {
-				if (!color.isTransparent()) { // Do not count transparency in for packing
+					tileColors.insert(color.cgbColor());
-					// Add the color to the proto-pal (if not full), and count it if it was unique.
-					if (tileColors.add(color.cgbColor())) {
-						++nbColorsInTile;
-					}
 				}
 			}
 		}
 
+		if (tileColors.size() > options.maxOpaqueColors()) {
+			fatal(
+			    "Tile at (%" PRIu32 ", %" PRIu32 ") has %zu colors, more than %" PRIu8 "!",
+			    tile.x,
+			    tile.y,
+			    tileColors.size(),
+			    options.maxOpaqueColors()
+			);
+		}
+
 		if (tileColors.empty()) {
 			// "Empty" proto-palettes screw with the packing process, so discard those
 			attrs.protoPaletteID = AttrmapEntry::transparent;
 			continue;
 		}
 
+		ProtoPalette protoPalette;
+		for (uint16_t cgbColor : tileColors) {
+			protoPalette.add(cgbColor);
+		}
+
 		// Insert the proto-palette, making sure to avoid overlaps
 		for (size_t n = 0; n < protoPalettes.size(); ++n) {
-			switch (tileColors.compare(protoPalettes[n])) {
+			switch (protoPalette.compare(protoPalettes[n])) {
 			case ProtoPalette::WE_BIGGER:
-				protoPalettes[n] = tileColors; // Override them
+				protoPalettes[n] = protoPalette; // Override them
 				// Remove any other proto-palettes that we encompass
 				// (Example [(0, 1), (0, 2)], inserting (0, 1, 2))
 				/*
@@ -1151,7 +1162,7 @@ void process() {
 				 * Investigation is necessary, especially if pathological cases are found.
 				 *
 				 * for (size_t i = protoPalettes.size(); --i != n;) {
-				 *     if (tileColors.compare(protoPalettes[i]) == ProtoPalette::WE_BIGGER) {
+				 *     if (protoPalette.compare(protoPalettes[i]) == ProtoPalette::WE_BIGGER) {
 				 *         protoPalettes.erase(protoPalettes.begin() + i);
 				 *     }
 				 * }
@@ -1168,17 +1179,6 @@ void process() {
 			}
 		}
 
-		if (nbColorsInTile > options.maxOpaqueColors()) {
-			fatal(
-			    "Tile at (%" PRIu32 ", %" PRIu32 ") has %" PRIu8 " opaque colors, more than %" PRIu8
-			    "!",
-			    tile.x,
-			    tile.y,
-			    nbColorsInTile,
-			    options.maxOpaqueColors()
-			);
-		}
-
 		attrs.protoPaletteID = protoPalettes.size();
 		if (protoPalettes.size() == AttrmapEntry::transparent) { // Check for overflow
 			fatal(
@@ -1186,7 +1186,7 @@ void process() {
 			    AttrmapEntry::transparent
 			);
 		}
-		protoPalettes.push_back(tileColors);
+		protoPalettes.push_back(protoPalette);
 continue_visiting_tiles:;
 	}
 

src/gfx/proto_palette.cpp

@@ -6,7 +6,7 @@
 
 #include "helpers.hpp"
 
-bool ProtoPalette::add(uint16_t color) {
+void ProtoPalette::add(uint16_t color) {
 	size_t i = 0;
 
 	// Seek the first slot greater than the new color
@@ -16,12 +16,12 @@ bool ProtoPalette::add(uint16_t color) {
 		++i;
 		if (i == _colorIndices.size()) {
 			// We reached the end of the array without finding the color, so it's a new one.
-			return true;
+			return;
 		}
 	}
 	// If we found it, great! Nothing else to do.
 	if (_colorIndices[i] == color) {
-		return false;
+		return;
 	}
 
 	// Swap entries until the end
@@ -30,12 +30,11 @@ bool ProtoPalette::add(uint16_t color) {
 		++i;
 		if (i == _colorIndices.size()) {
 			// The set is full, but doesn't include the new color.
-			return true;
+			return;
 		}
 	}
 	// Write that last one into the new slot
 	_colorIndices[i] = color;
-	return true;
 }
 
 ProtoPalette::ComparisonResult ProtoPalette::compare(ProtoPalette const &other) const {