Fix duplicated CGB color calculation in pal sorting

And also the inverted alpha channel condition causing the wrong colors
to be treated as transparent

include/gfx/convert.hpp

@@ -9,6 +9,51 @@
 #ifndef RGBDS_GFX_CONVERT_HPP
 #define RGBDS_GFX_CONVERT_HPP
 
+#include <assert.h>
+#include <stdint.h>
+
+#include "gfx/main.hpp"
+
+struct Rgba {
+	uint8_t red;
+	uint8_t green;
+	uint8_t blue;
+	uint8_t alpha;
+
+	Rgba(uint8_t r, uint8_t g, uint8_t b, uint8_t a) : red(r), green(g), blue(b), alpha(a) {}
+	Rgba(uint32_t rgba) : red(rgba), green(rgba >> 8), blue(rgba >> 16), alpha(rgba >> 24) {}
+
+	operator uint32_t() const {
+		auto shl = [](uint8_t val, unsigned shift) { return static_cast<uint32_t>(val) << shift; };
+		return shl(red, 0) | shl(green, 8) | shl(blue, 16) | shl(alpha, 24);
+	}
+	bool operator!=(Rgba const &other) const {
+		return static_cast<uint32_t>(*this) != static_cast<uint32_t>(other);
+	}
+
+	bool isGray() const { return red == green && green == blue; }
+
+	/**
+	 * CGB colors are RGB555, so we use bit 15 to signify that the color is transparent instead
+	 * Since the rest of the bits don't matter then, we return 0x8000 exactly.
+	 */
+	static constexpr uint16_t transparent = 0b1'00000'00000'00000;
+
+	static constexpr uint8_t transparency_threshold = 5; // TODO: adjust this
+	/**
+	 * Computes the equivalent CGB color, respects the color curve depending on options
+	 */
+	uint16_t cgbColor() const {
+		if (alpha > 0xFF - transparency_threshold)
+			return transparent;
+		if (options.useColorCurve) {
+			assert(!"TODO");
+		} else {
+			return (red >> 3) | (green >> 3) << 5 | (blue >> 3) << 10;
+		}
+	}
+};
+
 void process();
 
 #endif /* RGBDS_GFX_CONVERT_HPP */

src/gfx/convert.cpp

@@ -31,46 +31,6 @@
 #include "gfx/pal_sorting.hpp"
 #include "gfx/proto_palette.hpp"
 
-struct Rgba {
-	uint8_t red;
-	uint8_t green;
-	uint8_t blue;
-	uint8_t alpha;
-
-	Rgba(uint8_t r, uint8_t g, uint8_t b, uint8_t a) : red(r), green(g), blue(b), alpha(a) {}
-	Rgba(uint32_t rgba) : red(rgba), green(rgba >> 8), blue(rgba >> 16), alpha(rgba >> 24) {}
-
-	operator uint32_t() const {
-		auto shl = [](uint8_t val, unsigned shift) { return static_cast<uint32_t>(val) << shift; };
-		return shl(red, 0) | shl(green, 8) | shl(blue, 16) | shl(alpha, 24);
-	}
-	bool operator!=(Rgba const &other) const {
-		return static_cast<uint32_t>(*this) != static_cast<uint32_t>(other);
-	}
-
-	bool isGray() const { return red == green && green == blue; }
-
-	/**
-	 * CGB colors are RGB555, so we use bit 15 to signify that the color is transparent instead
-	 * Since the rest of the bits don't matter then, we return 0x8000 exactly.
-	 */
-	static constexpr uint16_t transparent = 0b1'00000'00000'00000;
-
-	static constexpr uint8_t transparency_threshold = 5; // TODO: adjust this
-	/**
-	 * Computes the equivalent CGB color, respects the color curve depending on options
-	 */
-	uint16_t cgbColor() const {
-		if (alpha < transparency_threshold)
-			return transparent;
-		if (options.useColorCurve) {
-			assert(!"TODO");
-		} else {
-			return (red >> 3) | (green >> 3) << 5 | (blue >> 3) << 10;
-		}
-	}
-};
-
 class ImagePalette {
 	// Use as many slots as there are CGB colors (plus transparency)
 	std::array<std::optional<Rgba>, 0x8001> _colors;
@@ -256,7 +216,8 @@ public:
 			options.verbosePrint("Embedded palette has %d colors: [", nbColors);
 			for (int i = 0; i < nbColors; ++i) {
 				auto const &color = embeddedPal[i];
-				options.verbosePrint("#%02x%02x%02x%s", color.red, color.green, color.blue,
+				options.verbosePrint("#%02x%02x%02x%02x%s", color.red, color.green, color.blue,
+				                     transparencyPal ? transparencyPal[i] : 0xFF,
 				                     i != nbColors - 1 ? ", " : "]\n");
 			}
 		} else {
@@ -464,7 +425,7 @@ static void outputTileData(Png const &png, DefaultInitVec<AttrmapEntry> const &a
                            std::vector<Palette> const &palettes,
                            DefaultInitVec<size_t> const &mappings) {
 	std::filebuf output;
-	output.open(options.tilemap, std::ios_base::out | std::ios_base::binary);
+	output.open(options.output, std::ios_base::out | std::ios_base::binary);
 
 	auto iter = attrmap.begin();
 	for (auto tile : png.visitAsTiles(options.columnMajor)) {

src/gfx/pal_sorting.cpp

@@ -7,6 +7,7 @@
 
 #include "helpers.h"
 
+#include "gfx/convert.hpp"
 #include "gfx/main.hpp"
 
 namespace sorting {
@@ -19,10 +20,12 @@ void indexed(std::vector<Palette> &palettes, int palSize, png_color const *palRG
 		std::sort(pal.begin(), pal.end(), [&](uint16_t lhs, uint16_t rhs) {
 			// Iterate through the PNG's palette, looking for either of the two
 			for (int i = 0; i < palSize; ++i) {
-				if (palAlpha && palAlpha[i])
-					continue;
 				auto const &c = palRGB[i];
-				uint16_t cgbColor = c.red >> 3 | (c.green >> 3) << 5 | (c.blue >> 3) << 10;
+				Rgba color(c.red, c.green, c.blue, palAlpha ? palAlpha[i] : 0xFF);
+				uint16_t cgbColor = color.cgbColor();
+				if (cgbColor == Rgba::transparent) {
+					continue;
+				}
 				// Return whether lhs < rhs
 				if (cgbColor == rhs) {
 					return false;