diff --git a/include/gfx/png.hpp b/include/gfx/png.hpp
index 6fa1fd30..87f15db6 100644
--- a/include/gfx/png.hpp
+++ b/include/gfx/png.hpp
@@ -14,6 +14,7 @@ struct Png {
 	uint32_t height = 0;
 	std::vector<Rgba> pixels{};
 	std::vector<Rgba> palette{};
+	bool isIndexed = false;
 
 	Png() {}
 	Png(char const *filename, std::streambuf &file);
diff --git a/src/gfx/pal_sorting.cpp b/src/gfx/pal_sorting.cpp
index 45918340..56712fd9 100644
--- a/src/gfx/pal_sorting.cpp
+++ b/src/gfx/pal_sorting.cpp
@@ -33,13 +33,7 @@ void sortIndexed(std::vector<Palette> &palettes, std::vector<Rgba> const &embPal
 					return true;
 				}
 			}
-			// A PNG image using PNG_COLOR_TYPE_RGB (2) or PNG_COLOR_TYPE_RGBA (6) can still
-			// contain a PLTE chunk. From the PNG spec: "If present, it provides a suggested set of
-			// from 1 to 256 colors to which the truecolor image can be quantized if the viewer
-			// cannot display truecolor directly." This means that an input image may have pixels
-			// with colors that are not in the PLTE chunk, so iterating through `embPal` may finish
-			// without returning. In that case, we compare `lhs` and `rhs` directly.
-			return lhs < rhs;
+			unreachable_(); // LCOV_EXCL_LINE
 		});
 	}
 }
diff --git a/src/gfx/png.cpp b/src/gfx/png.cpp
index 0f41e656..739a74d8 100644
--- a/src/gfx/png.cpp
+++ b/src/gfx/png.cpp
@@ -177,6 +177,7 @@ Png::Png(char const *filename, std::streambuf &file) {
 		break;
 	case PNG_COLOR_TYPE_PALETTE:
 		png_set_palette_to_rgb(png);
+		isIndexed = true; // This enables sorting generated palette colors by the PLTE chunk
 		break;
 	}
 
diff --git a/src/gfx/process.cpp b/src/gfx/process.cpp
index 2537790c..66b6c010 100644
--- a/src/gfx/process.cpp
+++ b/src/gfx/process.cpp
@@ -390,7 +390,14 @@ static std::pair<std::vector<size_t>, std::vector<Palette>>
 	// "Sort" colors in the generated palettes, see the man page for the flowchart
 	if (options.palSpecType == Options::DMG) {
 		sortGrayscale(palettes, image.colors.raw());
-	} else if (!image.png.palette.empty()) {
+	} else if (image.png.isIndexed) {
+		// A PNG image using PNG_COLOR_TYPE_RGB (2) or PNG_COLOR_TYPE_RGBA (6) can still
+		// contain a PLTE chunk. From the PNG spec: "If present, it provides a suggested set of
+		// from 1 to 256 colors to which the truecolor image can be quantized if the viewer
+		// cannot display truecolor directly." We only sort palette colors by the PLTE chunk's
+		// color order if the image uses PNG_COLOR_TYPE_PALETTE (3), since that guarantees every
+		// color used will also be in the embedded palette.
+		assume(!image.png.palette.empty());
 		warning(
 		    WARNING_EMBEDDED,
 		    "Sorting palette colors by PNG's embedded PLTE chunk without '-c/--colors embedded'"
diff --git a/test/gfx/rgb_with_plte.out.2bpp b/test/gfx/rgb_with_plte.out.2bpp
index 147b4e9a..6b014490 100644
Binary files a/test/gfx/rgb_with_plte.out.2bpp and b/test/gfx/rgb_with_plte.out.2bpp differ
diff --git a/test/gfx/rgb_with_plte.out.pal b/test/gfx/rgb_with_plte.out.pal
index 98c4fa00..9f6fe38b 100644
Binary files a/test/gfx/rgb_with_plte.out.pal and b/test/gfx/rgb_with_plte.out.pal differ