diff --git a/src/gfx/pal_packing.cpp b/src/gfx/pal_packing.cpp
index 60cece66..2c5ebae7 100644
--- a/src/gfx/pal_packing.cpp
+++ b/src/gfx/pal_packing.cpp
@@ -362,7 +362,19 @@ std::tuple<DefaultInitVec<size_t>, size_t>
 	DefaultInitVec<size_t> sortedProtoPalIDs(protoPalettes.size());
 	sortedProtoPalIDs.clear();
 	for (size_t i = 0; i < protoPalettes.size(); ++i) {
-		sortedProtoPalIDs.insert(std::lower_bound(RANGE(sortedProtoPalIDs), i), i);
+		sortedProtoPalIDs.insert(
+		    std::lower_bound(
+		        RANGE(sortedProtoPalIDs),
+		        i,
+		        [&protoPalettes](size_t left, size_t right) {
+			        ProtoPalette const &lhs = protoPalettes[left];
+			        ProtoPalette const &rhs = protoPalettes[right];
+			        return lhs.size()
+			               > rhs.size(); // We want the proto-pals to be sorted *largest first*!
+		        }
+		    ),
+		    i
+		);
 	}
 	// Begin with all proto-palettes queued up for insertion
 	std::queue<ProtoPalAttrs> queue(std::deque<ProtoPalAttrs>(RANGE(sortedProtoPalIDs)));
diff --git a/test/gfx/at-file-ref.err b/test/gfx/at-file-ref.err
index b0358a13..3067cdc0 100644
--- a/test/gfx/at-file-ref.err
+++ b/test/gfx/at-file-ref.err
@@ -6,9 +6,9 @@ error: Failed to fit tile colors [$6c8a, $7f55] in specified palettes
 error: Failed to fit tile colors [$6c8a] in specified palettes
 error: Failed to fit tile colors [$7fff] in specified palettes
 note: The following palettes were specified:
-        [$3f65, $36b3, $262a, $50b0]
         [$5f77, $213d, $41a6, $40ee]
-        [$5f77, $267c, $213d, $40ee]
+        [$3f65, $36b3, $262a, $50b0]
         [$53c3, $3f65, $36b3]
-        [$267c, $41a6]
+        [$5f77, $267c, $41a6]
+        [$267c, $213d, $40ee]
 Conversion aborted after 7 errors