Remove the need to explicitly list external tests

I like having a single source of truth!
2026-08-19 21:34:31 +00:00 · 2026-07-02 10:39:59 -04:00
parent d59081a918
commit 5e8a6cde20
12 changed files with 71 additions and 64 deletions
@@ -233,25 +233,7 @@ Each one defines the parameters for a real external project that builds using RG
   While the test suite supports any Make target name, only
   [Make](//gnu.org/software/make) is currently supported, and the Makefile must
   support a `RGBDS` variable to use a non-system RGBDS directory.
-2. Add the project to `test/fetch-test-deps.sh`: add a new line at the bottom,
-   following the existing pattern:
-   
-   ```sh
-   . external/<name>.cfg && action
-   ```
-3. Add the project to `test/run-tests.sh`: add a new line at the bottom,
-   following the existing pattern:
-   
-   ```sh
-   ./external/test.sh <name>
-   ```
-4. Add the project to `CMakeLists.txt`: add it to the `foreach(project ...)`
-   list, and add new lines at the bottom, following the existing pattern:
-   
-   ```cmake
-   set_tests_properties(<name> PROPERTIES LABELS "<name>;external;free"
-                               FIXTURES_REQUIRED "free-repos")
-   ```
+2. Create the new `.cfg` file, following the existing ones.

 ## Container images

@@ -39,27 +39,33 @@ set_tests_properties(fetch-nonfree-deps PROPERTIES FIXTURES_SETUP "external-repo
                                                   FIXTURES_REQUIRED "free-repos"
                                                   LABELS "external;nonfree")

-foreach(project "pokecrystal" "pokered" "ladx" "ucity" "libbet" "sameboy" "gb-starter-kit")
+file(GLOB ext_projects "external/*.cfg") # This will not be re-computed if a new project is added! You must manually reconfigure.
+foreach(cfg_file IN LISTS ext_projects)
+  cmake_path(GET cfg_file STEM LAST_ONLY project) # Extract the project's name from the config file's name.
+
+  # Parse the config file, using it to set some variables for more convenient access.
+  file(STRINGS "${cfg_file}" proj_props REGEX "^[^# \t]*[^#]") # Ignore comments and empty lines.
+  foreach(line LIST proj_props)
+    string(REGEX MATCH "^[ \t]*EXT_TEST_([A-Za-z0-9_]+)=(.+)" matched "${line}")
+    set(${project}_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}")
+  endforeach()
+
  add_test(NAME "${project}"
           COMMAND bash -- external/test.sh ${project}
           WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
  set_tests_properties(${project} PROPERTIES DEPENDS "rgbasm;rgblink;rgbfix;rgbgfx" # Only attempt building whole projects if each tool passes muster on its own.
-                                             PROCESSORS 4)
+                                             PROCESSORS 4
+                                             LABELS "external;${project}")
+
+  # Wire it up according to whether it's free (as in speech) or not.
+  if(${project}_IS_NONFREE)
+    set_property(TEST "${project}" APPEND PROPERTY LABELS "nonfree")
+    set_tests_properties("${project}" PROPERTIES FIXTURES_REQUIRED "external-repos")
+  else()
+    set_property(TEST "${project}" APPEND PROPERTY LABELS "free")
+    set_tests_properties("${project}" PROPERTIES FIXTURES_REQUIRED "free-repos")
+  endif()
 endforeach()
-set_tests_properties(pokecrystal    PROPERTIES LABELS "pokecrystal;external;nonfree"
-                                               FIXTURES_REQUIRED "external-repos")
-set_tests_properties(pokered        PROPERTIES LABELS "pokered;external;nonfree"
-                                               FIXTURES_REQUIRED "external-repos")
-set_tests_properties(ladx           PROPERTIES LABELS "ladx;external;nonfree"
-                                               FIXTURES_REQUIRED "external-repos")
-set_tests_properties(ucity          PROPERTIES LABELS "ucity;external;free"
-                                               FIXTURES_REQUIRED "free-repos")
-set_tests_properties(libbet         PROPERTIES LABELS "libbet;external;free"
-                                               FIXTURES_REQUIRED "free-repos")
-set_tests_properties(sameboy        PROPERTIES LABELS "sameboy;external;free"
-                                               FIXTURES_REQUIRED "free-repos")
-set_tests_properties(gb-starter-kit PROPERTIES LABELS "gb-starter-kit;external;free"
-                                               FIXTURES_REQUIRED "free-repos")

 # gb-starter kit fails with any `make` on Windows: https://codeberg.org/ISSOtm/gb-starter-kit/issues/1
 # gb-starter-kit fails with macOS/BSD `make`: https://codeberg.org/ISSOtm/gb-starter-kit/issues/29
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=ISSOtm
 EXT_TEST_REPO=gb-starter-kit
 EXT_TEST_DOMAIN=codeberg.org
 EXT_TEST_COMMIT=74b647d62ff74b40d2b52e585cbebe148463212e
+EXT_TEST_IS_NONFREE=false
+
 EXT_TEST_TARGET=all
 EXT_TEST_FILE=bin/boilerplate.gb
 EXT_TEST_HASH=b4f130169ba73284e0d0e71b53e7baa4eca2f7fe
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=zladx
 EXT_TEST_REPO=LADX-Disassembly
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=c77af4473e7a877c68e1de34a2aaf80e9076dc35
+EXT_TEST_IS_NONFREE=true
+
 EXT_TEST_TARGET=default
 EXT_TEST_FILE=azle.gbc
 EXT_TEST_HASH=d90ac17e9bf17b6c61624ad9f05447bdb5efc01a
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=pinobatch
 EXT_TEST_REPO=libbet
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=e42c0036b18e6e715987b88b4973389b283974c9
+EXT_TEST_IS_NONFREE=false
+
 EXT_TEST_TARGET=all
 EXT_TEST_FILE=libbet.gb
 EXT_TEST_HASH=f117089aa056600e2d404bbcbac96b016fc64611
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=pret
 EXT_TEST_REPO=pokecrystal
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=2bbb15675de0d2bbebc8cc9978f5c7fb15bc73b9
+EXT_TEST_IS_NONFREE=true
+
 EXT_TEST_TARGET=compare
 EXT_TEST_FILE=pokecrystal.gbc
 EXT_TEST_HASH=f4cd194bdee0d04ca4eac29e09b8e4e9d818c133
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=pret
 EXT_TEST_REPO=pokered
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=0555b42dc0ceffaae613e97cc0cf2e8c0b45013c
+EXT_TEST_IS_NONFREE=true
+
 EXT_TEST_TARGET=compare
 EXT_TEST_FILE=pokered.gbc
 EXT_TEST_HASH=ea9bcae617fdf159b045185467ae58b2e4a48b9a
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=LIJI32
 EXT_TEST_REPO=SameBoy
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=2f4a6f231ec40ecfc0ab7df0a09eb932e7ccddec
+EXT_TEST_IS_NONFREE=false
+
 EXT_TEST_TARGET=bootroms
 EXT_TEST_FILE=build/bin/BootROMs/cgb_boot.bin
 EXT_TEST_HASH=113903775a9d34b798c2f8076672da6626815a91
@@ -15,7 +15,7 @@ if [ ! -f "external/$1.cfg" ]; then
 	exit 1
 fi

-# Sourcing "external/$1.cfg" defines `EXT_TEST_*` values used below.
+# Sourcing "$1.cfg" defines `EXT_TEST_*` variables used below.
 . "external/$1.cfg"

 if ! cd "$EXT_TEST_REPO"; then
@@ -2,6 +2,8 @@ EXT_TEST_OWNER=AntonioND
 EXT_TEST_REPO=ucity
 EXT_TEST_DOMAIN=github.com
 EXT_TEST_COMMIT=d1880a2a112d7c26f16c0fc06a15b6c32fdc9137
+EXT_TEST_IS_NONFREE=false
+
 EXT_TEST_TARGET=all
 EXT_TEST_FILE=ucity.gbc
 EXT_TEST_HASH=5f026649611c9606ce0bf70dc1552e054e7df5bc
@@ -18,7 +18,6 @@ EOF
 # in what util-linux `getopt` calls `GETOPT_COMPATIBLE` mode
 nonfree=true
 actionname=
-osname=
 while [[ $# -gt 0 ]]; do
 	case "$1" in
 		-h|--help)
@@ -73,13 +72,13 @@ case "$actionname" in
 		}
 esac

-# Sourcing each "external/*.cfg" file defines `EXT_TEST_*` values used by the `action` functions.
-if "$nonfree"; then
-	. external/pokecrystal.cfg && action
-	. external/pokered.cfg     && action
-	. external/ladx.cfg        && action
-fi
-. external/ucity.cfg          && action
-. external/libbet.cfg         && action
-. external/sameboy.cfg        && action
-. external/gb-starter-kit.cfg && action
+# Since each iteration sources variables into the shell itself,
+#   we do that in a subshell so that they don't "leak" out.
+for cfg in external/*.cfg; do (
+	# The sourced file defines `EXT_TEST_*` variables that get used by `action`.
+	. "$cfg"
+	# Only run a nonfree action if nonfree tests are opted into.
+	if ! $EXT_TEST_IS_NONFREE || $nonfree; then
+		action
+	fi
+); done
@@ -62,7 +62,7 @@ while [[ $# -gt 0 ]]; do
 	shift
 done

-if ! ("$internal" || "$external"); then
+if ! "$internal" && ! "$external"; then
 	echo "Specifying --only-internal with --only-external is a contradiction"
 	false
 fi
@@ -102,20 +102,26 @@ if ! "$external"; then
 	exit
 fi

-# Test some significant external projects that use RGBDS
-# When adding new ones, don't forget to add them to the .gitignore!
+# Test some significant external projects that use RGBDS.
+# Note that they are intentionally sequenced after the internal tests, since
+# failures reported by internal tests will be easier to debug than external ones.
+cd external
+# Each iteration is isolated in a (subshell) so the sourced cfg variables don't "leak" out.
+for cfg in *.cfg; do (
+	# Sourcing "$cfg" defines the `EXT_TEST_IS_NONFREE` variable used below.
+	. "$cfg"
+	test_name="${cfg%.cfg}"

-if "$nonfree"; then
-	./external/test.sh pokecrystal
-	./external/test.sh pokered
-	./external/test.sh ladx
-fi
-./external/test.sh ucity
-./external/test.sh libbet
-./external/test.sh sameboy
-# gb-starter kit fails with any `make` on Windows: https://codeberg.org/ISSOtm/gb-starter-kit/issues/1
-# gb-starter-kit fails with macOS/BSD `make`: https://codeberg.org/ISSOtm/gb-starter-kit/issues/29
-case "${osname%%-*}" in
-	windows | macos | *bsd) ;;
-	*) ./external/test.sh gb-starter-kit
-esac
+	if [ "$test_name" = gb-starter-kit ]; then
+		# gb-starter kit fails with any `make` on Windows: https://codeberg.org/ISSOtm/gb-starter-kit/issues/1
+		# gb-starter-kit fails with macOS/BSD `make`: https://codeberg.org/ISSOtm/gb-starter-kit/issues/29
+		case "${osname%%-*}" in
+			windows | macos | *bsd) exit 0 ;; # Note that this only exits the subshell!
+		esac
+	fi
+
+	# Run nonfree tests only if they are opted into.
+	if ! "$EXT_TEST_IS_NONFREE" || "$nonfree"; then
+		./test.sh "$test_name"
+	fi
+); done