diff --git a/include/gfx/proto_palette.hpp b/include/gfx/proto_palette.hpp
index 33bf4db8..101a5ac5 100644
--- a/include/gfx/proto_palette.hpp
+++ b/include/gfx/proto_palette.hpp
@@ -17,6 +17,7 @@
 class ProtoPalette {
 	// Up to 4 colors, sorted, and where SIZE_MAX means the slot is empty
 	// (OK because it's not a valid color index)
+	// Sorting is done on the raw numerical values to lessen `compare`'s complexity
 	std::array<uint16_t, 4> _colorIndices{UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX};
 
 public:
diff --git a/src/gfx/proto_palette.cpp b/src/gfx/proto_palette.cpp
index 91ec8b92..641406be 100644
--- a/src/gfx/proto_palette.cpp
+++ b/src/gfx/proto_palette.cpp
@@ -10,6 +10,7 @@
 
 #include <algorithm>
 #include <array>
+#include <cassert>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -41,6 +42,10 @@ bool ProtoPalette::add(uint16_t color) {
 }
 
 ProtoPalette::ComparisonResult ProtoPalette::compare(ProtoPalette const &other) const {
+	// This works because the sets are sorted numerically
+	assert(std::is_sorted(_colorIndices.begin(), _colorIndices.end()));
+	assert(std::is_sorted(other._colorIndices.begin(), other._colorIndices.end()));
+
 	auto ours = _colorIndices.begin(), theirs = other._colorIndices.begin();
 	bool weBigger = true, theyBigger = true;
 
@@ -56,8 +61,8 @@ ProtoPalette::ComparisonResult ProtoPalette::compare(ProtoPalette const &other)
 			weBigger = false;
 		}
 	}
-	weBigger &= ours == _colorIndices.end();
-	theyBigger &= theirs == other._colorIndices.end();
+	weBigger &= theirs == other._colorIndices.end();
+	theyBigger &= ours == _colorIndices.end();
 
 	return theyBigger ? THEY_BIGGER : (weBigger ? WE_BIGGER : NEITHER);
 }