gbdev/rgbds
1
0
Fork 0
mirror of https://github.com/gbdev/rgbds.git synced 2026-07-02 05:58:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the test harness for external repo tests (#1994)

Browse Source 
- Use CTest labels to filter tests ("internal"/"external", "free"/"nonfree",
  and individual tool+project names) instead of defining `TESTS_RUN_NONFREE`
- Allow each external test to run independently in CTest
- Remove the unused no-op `fetch-test-deps.sh --only-internal` option
This commit is contained in:
Rangi
2026-06-30 10:19:41 -04:00
committed by GitHub
parent b252877b05
commit a829e6c067
13 changed files with 189 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/CMakeLists.txt
+36 -20
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: MIT
option(TESTS_RUN_NONFREE "Run tests that build nonfree codebases." ON)
set(TESTS_OS_NAME "" CACHE STRING "Skip running tests known to fail on this OS.")
add_executable(randtilegen gfx/randtilegen.cpp)
@@ -18,35 +17,52 @@ foreach(prog "randtilegen" "rgbgfx_test")
COMMAND_EXPAND_LISTS VERBATIM)
endforeach()
set(ONLY_FREE)
if(NOT TESTS_RUN_NONFREE)
set(ONLY_FREE "--only-free")
endif()
set(OS_NAME)
if(NOT TESTS_OS_NAME STREQUAL "")
set(OS_NAME "--os" "${TESTS_OS_NAME}")
endif()
foreach(component "asm" "link" "fix" "gfx")
add_test(NAME "rgb${component}"
COMMAND bash -- test.sh
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${component}")
set_tests_properties("rgb${component}" PROPERTIES REQUIRED_FILES "$<TARGET_FILE:rgb${component}>"
PROCESSORS 1
LABELS "rgb${component}")
LABELS "rgb${component};internal;free")
endforeach()
set_tests_properties(rgbgfx PROPERTIES REQUIRED_FILES "$<TARGET_FILE:rgbgfx>;$<TARGET_FILE:randtilegen>;$<TARGET_FILE:rgbgfx_test>")
add_test(NAME fetch-test-deps
COMMAND bash -- fetch-test-deps.sh ${ONLY_FREE}
COMMAND bash -- fetch-test-deps.sh --only-free
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
set_tests_properties(fetch-test-deps PROPERTIES FIXTURES_SETUP "external-repos"
set_tests_properties(fetch-test-deps PROPERTIES FIXTURES_SETUP "free-repos"
LABELS "external")
add_test(NAME external
COMMAND bash -- run-tests.sh --only-external ${ONLY_FREE} ${OS_NAME}
add_test(NAME fetch-nonfree-deps
COMMAND bash -- fetch-test-deps.sh
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
set_tests_properties(external PROPERTIES DEPENDS "rgbasm;rgblink;rgbfix;rgbgfx" # Only attempt building whole projects if each tool passes muster on its own.
PROCESSORS 4
FIXTURES_REQUIRED "external-repos"
LABELS "external") # Allow filtering out external tests.
set_tests_properties(fetch-nonfree-deps PROPERTIES FIXTURES_SETUP "external-repos"
FIXTURES_REQUIRED "free-repos"
LABELS "external;nonfree")
foreach(project "pokecrystal" "pokered" "ladx" "ucity" "libbet" "sameboy" "gb-starter-kit")
add_test(NAME "${project}"
COMMAND bash -- external/test.sh ${project}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
set_tests_properties(${project} PROPERTIES DEPENDS "rgbasm;rgblink;rgbfix;rgbgfx" # Only attempt building whole projects if each tool passes muster on its own.
PROCESSORS 4)
endforeach()
set_tests_properties(pokecrystal PROPERTIES LABELS "pokecrystal;external;nonfree"
FIXTURES_REQUIRED "external-repos")
set_tests_properties(pokered PROPERTIES LABELS "pokered;external;nonfree"
FIXTURES_REQUIRED "external-repos")
set_tests_properties(ladx PROPERTIES LABELS "ladx;external;nonfree"
FIXTURES_REQUIRED "external-repos")
set_tests_properties(ucity PROPERTIES LABELS "ucity;external;free"
FIXTURES_REQUIRED "free-repos")
set_tests_properties(libbet PROPERTIES LABELS "libbet;external;free"
FIXTURES_REQUIRED "free-repos")
set_tests_properties(sameboy PROPERTIES LABELS "sameboy;external;free"
FIXTURES_REQUIRED "free-repos")
set_tests_properties(gb-starter-kit PROPERTIES LABELS "gb-starter-kit;external;free"
FIXTURES_REQUIRED "free-repos")
# gb-starter kit fails with any `make` on Windows: https://codeberg.org/ISSOtm/gb-starter-kit/issues/1
# gb-starter-kit fails with macOS/BSD `make`: https://codeberg.org/ISSOtm/gb-starter-kit/issues/29
if(TESTS_OS_NAME MATCHES "^(windows|macos|.*bsd)(-.*)?")
set_tests_properties(gb-starter-kit PROPERTIES DISABLED TRUE)
endif()
test/external/gb-starter-kit.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=ISSOtm
EXTERNAL_TEST_REPO=gb-starter-kit
EXTERNAL_TEST_DOMAIN=codeberg.org
EXTERNAL_TEST_COMMIT=74b647d62ff74b40d2b52e585cbebe148463212e
EXTERNAL_TEST_TARGET=all
EXTERNAL_TEST_FILE=bin/boilerplate.gb
EXTERNAL_TEST_HASH=b4f130169ba73284e0d0e71b53e7baa4eca2f7fe
test/external/ladx.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=zladx
EXTERNAL_TEST_REPO=LADX-Disassembly
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=c77af4473e7a877c68e1de34a2aaf80e9076dc35
EXTERNAL_TEST_TARGET=default
EXTERNAL_TEST_FILE=azle.gbc
EXTERNAL_TEST_HASH=d90ac17e9bf17b6c61624ad9f05447bdb5efc01a
test/external/libbet.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=pinobatch
EXTERNAL_TEST_REPO=libbet
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=e42c0036b18e6e715987b88b4973389b283974c9
EXTERNAL_TEST_TARGET=all
EXTERNAL_TEST_FILE=libbet.gb
EXTERNAL_TEST_HASH=f117089aa056600e2d404bbcbac96b016fc64611
test/external/pokecrystal.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=pret
EXTERNAL_TEST_REPO=pokecrystal
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=2bbb15675de0d2bbebc8cc9978f5c7fb15bc73b9
EXTERNAL_TEST_TARGET=compare
EXTERNAL_TEST_FILE=pokecrystal.gbc
EXTERNAL_TEST_HASH=f4cd194bdee0d04ca4eac29e09b8e4e9d818c133
test/external/pokered.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=pret
EXTERNAL_TEST_REPO=pokered
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=0555b42dc0ceffaae613e97cc0cf2e8c0b45013c
EXTERNAL_TEST_TARGET=compare
EXTERNAL_TEST_FILE=pokered.gbc
EXTERNAL_TEST_HASH=ea9bcae617fdf159b045185467ae58b2e4a48b9a
test/external/sameboy.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=LIJI32
EXTERNAL_TEST_REPO=SameBoy
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=2f4a6f231ec40ecfc0ab7df0a09eb932e7ccddec
EXTERNAL_TEST_TARGET=bootroms
EXTERNAL_TEST_FILE=build/bin/BootROMs/cgb_boot.bin
EXTERNAL_TEST_HASH=113903775a9d34b798c2f8076672da6626815a91
test/external/test.sh
Vendored Executable
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
export LC_ALL=C
# Game Boy release date, 1989-04-21T12:34:56Z (for reproducible test results)
export SOURCE_DATE_EPOCH=609165296
cd "$(dirname "$0")/.."
RGBDS_PATH="RGBDS=../../"
if [ ! -f "external/$1.cfg" ]; then
echo >&2 'External test file '"$1"'.cfg does not exist'
exit 1
fi
# Sourcing "external/$1.cfg" defines `EXTERNAL_TEST_*` values used below.
. "external/$1.cfg"
if ! cd "$EXTERNAL_TEST_REPO"; then
echo >&2 'Please fetch test deps before running any external test'
exit 1
fi
make clean $RGBDS_PATH
make -j4 "$EXTERNAL_TEST_TARGET" $RGBDS_PATH
hash="$(sha1sum -b "$EXTERNAL_TEST_FILE" | head -c 40)"
if [ "$hash" != "$EXTERNAL_TEST_HASH" ]; then
cat >&2 <<EOM
error: "$EXTERNAL_TEST_FILE" checksum did not match!
Expected $EXTERNAL_TEST_HASH,
got $hash
EOM
exit 1
fi
test/external/ucity.cfg
Vendored
+7
View File
@@ -0,0 +1,7 @@
EXTERNAL_TEST_OWNER=AntonioND
EXTERNAL_TEST_REPO=ucity
EXTERNAL_TEST_DOMAIN=github.com
EXTERNAL_TEST_COMMIT=d1880a2a112d7c26f16c0fc06a15b6c32fdc9137
EXTERNAL_TEST_TARGET=all
EXTERNAL_TEST_FILE=ucity.gbc
EXTERNAL_TEST_HASH=5f026649611c9606ce0bf70dc1552e054e7df5bc
test/fetch-test-deps.sh
+26 -32
View File
@@ -5,20 +5,18 @@ cd "$(dirname "$0")"
usage() {
cat <<"EOF"
Downloads source code of Game Boy programs used as RGBDS test cases.
Downloads source code of Game Boy project repos used as RGBDS test cases.
Options:
-h, --help show this help message
--only-free download only freely licensed codebases
--only-internal do not download any codebases
--get-hash print programs' commit hashes instead of downloading them
--get-paths print programs' GitHub paths instead of downloading them
-h, --help show this help message
--only-free download only freely licensed codebases
--get-hash print repos' commit hashes instead of downloading them
--get-paths print repos' clone paths instead of downloading them
EOF
}
# Parse options in pure Bash because macOS `getopt` is stuck
# in what util-linux `getopt` calls `GETOPT_COMPATIBLE` mode
nonfree=true
external=true
actionname=
osname=
while [[ $# -gt 0 ]]; do
@@ -30,9 +28,6 @@ while [[ $# -gt 0 ]]; do
--only-free)
nonfree=false
;;
--only-internal)
external=false
;;
--get-hash|--get-paths)
actionname="$1"
;;
@@ -49,43 +44,42 @@ done
case "$actionname" in
--get-hash)
action() { # _ _ repo commit
printf "%s@%s-" "$3" "$4"
action() {
printf "%s@%s-" "$EXTERNAL_TEST_REPO" "$EXTERNAL_TEST_COMMIT"
}
;;
--get-paths)
action() { # _ _ repo _
printf "test/%s," "$3"
action() {
printf "test/%s," "$EXTERNAL_TEST_REPO"
}
;;
*)
echo "Fetching test dependency repositories"
action() { # domain owner repo commit
if [ ! -d "$3" ]; then
git clone "https://$1/$2/$3.git" --revision="$4" --depth=1 --recursive --shallow-submodules --config advice.detachedHead=false
action() {
if [ ! -d "$EXTERNAL_TEST_REPO" ]; then
git clone "https://$EXTERNAL_TEST_DOMAIN/$EXTERNAL_TEST_OWNER/$EXTERNAL_TEST_REPO.git" \
--revision="$EXTERNAL_TEST_COMMIT" --depth=1 --recursive --shallow-submodules \
--config advice.detachedHead=false
fi
pushd "$3"
git checkout --force --detach "$4" --
if [ -f "../patches/$3.patch" ]; then
git apply --ignore-whitespace "../patches/$3.patch"
pushd "$EXTERNAL_TEST_REPO"
git checkout --force --detach "$EXTERNAL_TEST_COMMIT" --
if [ -f "../patches/$EXTERNAL_TEST_REPO.patch" ]; then
git apply --ignore-whitespace "../patches/$EXTERNAL_TEST_REPO.patch"
fi
popd
}
esac
if ! "$external"; then
exit
fi
# Sourcing each "external/*.cfg" file defines `EXTERNAL_TEST_*` values used by the `action` functions.
if "$nonfree"; then
action github.com pret pokecrystal 2bbb15675de0d2bbebc8cc9978f5c7fb15bc73b9
action github.com pret pokered 0555b42dc0ceffaae613e97cc0cf2e8c0b45013c
action github.com zladx LADX-Disassembly c77af4473e7a877c68e1de34a2aaf80e9076dc35
. external/pokecrystal.cfg && action
. external/pokered.cfg && action
. external/ladx.cfg && action
fi
action github.com AntonioND ucity d1880a2a112d7c26f16c0fc06a15b6c32fdc9137
action github.com pinobatch libbet e42c0036b18e6e715987b88b4973389b283974c9
action github.com LIJI32 SameBoy 2f4a6f231ec40ecfc0ab7df0a09eb932e7ccddec
action codeberg.org ISSOtm gb-starter-kit 74b647d62ff74b40d2b52e585cbebe148463212e
. external/ucity.cfg && action
. external/libbet.cfg && action
. external/sameboy.cfg && action
. external/gb-starter-kit.cfg && action
test/run-tests.sh
+7 -25
View File
@@ -39,7 +39,6 @@ while [[ $# -gt 0 ]]; do
;;
--only-internal)
external=false
FETCH_TEST_DEPS="$FETCH_TEST_DEPS --only-internal"
;;
--only-external)
internal=false
@@ -109,35 +108,18 @@ fi
# Test some significant external projects that use RGBDS
# When adding new ones, don't forget to add them to the .gitignore!
# When updating subprojects, change the commit being checked out, and set the `shallow-since`
# to the day before, to reduce the amount of refs being transferred and thus speed up CI.
test_downstream() { # owner repo make-target build-file build-hash
if ! pushd "$2"; then
echo >&2 'Please run `'"$FETCH_TEST_DEPS"'` before running the test suite'
return 1
fi
make clean $RGBDS_PATH
make -j4 "$3" $RGBDS_PATH
hash="$(sha1sum -b "$4" | head -c 40)"
if [ "$hash" != "$5" ]; then
echo >&2 'SHA-1 hash of '"$4"' did not match: '"$hash"
return 1
fi
popd
}
if "$nonfree"; then
test_downstream pret pokecrystal compare pokecrystal.gbc f4cd194bdee0d04ca4eac29e09b8e4e9d818c133
test_downstream pret pokered compare pokered.gbc ea9bcae617fdf159b045185467ae58b2e4a48b9a
test_downstream zladx LADX-Disassembly default azle.gbc d90ac17e9bf17b6c61624ad9f05447bdb5efc01a
./external/test.sh pokecrystal
./external/test.sh pokered
./external/test.sh ladx
fi
test_downstream AntonioND ucity all ucity.gbc 5f026649611c9606ce0bf70dc1552e054e7df5bc
test_downstream pinobatch libbet all libbet.gb f117089aa056600e2d404bbcbac96b016fc64611
test_downstream LIJI32 SameBoy bootroms build/bin/BootROMs/cgb_boot.bin 113903775a9d34b798c2f8076672da6626815a91
./external/test.sh ucity
./external/test.sh libbet
./external/test.sh sameboy
# gb-starter kit fails with any `make` on Windows: https://codeberg.org/ISSOtm/gb-starter-kit/issues/1
# gb-starter-kit fails with macOS/BSD `make`: https://codeberg.org/ISSOtm/gb-starter-kit/issues/29
case "${osname%%-*}" in
windows | macos | *bsd) ;;
*) test_downstream ISSOtm gb-starter-kit all bin/boilerplate.gb b4f130169ba73284e0d0e71b53e7baa4eca2f7fe;;
*) ./external/test.sh gb-starter-kit
esac