Break down CTest cases into each test suite (#1931)

This allows parallelising the individual programs'
test suites, which can save a few minutes!

.github/workflows/testing.yml

@@ -6,11 +6,14 @@ on:
 env:
   CLICOLOR_FORCE: 1 # Tells CMake to have colored output.
   CMAKE_COLOR_DIAGNOSTICS: ON # Tells CMake-generated build systems to have colored output.
+  CMAKE_CONFIG_TYPE: Debug # `cmake --build` now implies `--config Debug`.
   # Approximate number of CPU cores in GitHub's runners as of 2026-03-18:
   # https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories
   CMAKE_BUILD_PARALLEL_LEVEL: 4 # `cmake --build` now implies `--parallel 4`.
   CMAKE_INSTALL_PARALLEL_LEVEL: 4 # `cmake --install` now implies `--parallel 4`.
-  CMAKE_CONFIG_TYPE: Debug # `cmake --build` now implies `--config Debug`.
+  CTEST_PARALLEL_LEVEL: 0 # `ctest` now implies `--parallel 0` (number of logical CPUs).
+  CTEST_NO_TESTS_ACTION: error # Make CTest fail if it cannot find any tests. (That should never happen.)
+  CTEST_OUTPUT_ON_FAILURE: ON # CTest reports test program output on failure.
 
   GIT_CONFIG_COUNT: 1
   GIT_CONFIG_KEY_0: color.ui
@@ -96,7 +99,7 @@ jobs:
       - name: Run tests using CTest
         if: matrix.buildsys == 'cmake'
         run: |
-          ctest --test-dir build --verbose
+          ctest --test-dir build --schedule-random
 
   macos-static:
     runs-on: macos-14
@@ -168,7 +171,8 @@ jobs:
           enableCrossOsArchive: true # Currently only used on Windows, but the contents are OS-agnostic.
       - name: Build Windows binaries
         run: | # ASan seems to be broken on Windows, so we disable it.
-          cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}" -DSANITIZERS=OFF
+          cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DSANITIZERS=OFF \
+                -DTESTS_OS_NAME=${{ matrix.os }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}"
           cmake --build build
       - name: Package binaries
         working-directory: build
@@ -201,9 +205,9 @@ jobs:
       - name: Install test dependency dependencies
         run: |
           test/fetch-test-deps.sh --get-deps ${{ matrix.os }}
-      - name: Run tests
+      - name: Run tests using CTest
         run: |
-          test/run-tests.sh --os ${{ matrix.os }}
+          ctest --test-dir build --schedule-random -C Debug
 
   windows-mingw-build:
     strategy:
@@ -353,13 +357,17 @@ jobs:
       - name: Build & test using CMake on FreeBSD
         uses: vmactions/freebsd-vm@v1
         with:
-          envs: CLICOLOR_FORCE CMAKE_COLOR_DIAGNOSTICS CMAKE_BUILD_PARALLEL_LEVEL CMAKE_INSTALL_PARALLEL_LEVEL
+          envs: >-
+            CLICOLOR_FORCE CMAKE_COLOR_DIAGNOSTICS CMAKE_CONFIG_TYPE
+            CMAKE_BUILD_PARALLEL_LEVEL CMAKE_INSTALL_PARALLEL_LEVEL CTEST_PARALLEL_LEVEL
+            CTEST_NO_TESTS_ACTION CTEST_OUTPUT_ON_FAILURE
+            GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0
           release: "14.3"
           usesh: true
           prepare: |
             .github/scripts/install_deps.sh freebsd
           run: | # Leak detection is not supported on FreeBSD, so disable it.
-            cmake -S . -B build --preset develop -DTESTS_RUN_EXTERNAL=OFF -DTESTS_OS_NAME=freebsd
+            cmake -S . -B build --preset develop -DTESTS_OS_NAME=freebsd
             cmake --build build --verbose
+            ASAN_OPTIONS=detect_leaks=0 ctest --test-dir build --schedule-random --label-exclude external
             cmake --install build --verbose
-            ASAN_OPTIONS=detect_leaks=0 ctest --test-dir build --verbose

CMakeLists.txt

@@ -136,11 +136,17 @@ endif()
 include(FetchContent)
 include(cmake/deps.cmake)
 
+if(NOT DEFINED ZLIB_BUILD_TESTING) # Unless overridden (e.g. in the cache), we don't care about zlib's tests.
+  set(ZLIB_BUILD_TESTING OFF)
+endif()
 FetchContent_MakeAvailable(ZLIB)
 if(NOT DEFINED ZLIB_INCLUDE_DIRS)
   set(ZLIB_INCLUDE_DIRS "${zlib_BINARY_DIR};${zlib_SOURCE_DIR}") # libpng's `genout` script relies on this variable to be set.
 endif()
 
+if(NOT DEFINED PNG_TESTS) # Unless overridden (e.g. in the cache), we don't care about libpng's tests.
+  set(PNG_TESTS OFF)
+endif()
 FetchContent_MakeAvailable(PNG)
 if(NOT TARGET PNG::PNG)
   if(PNG_SHARED)

docs/CONTRIBUTING.md

@@ -65,7 +65,7 @@ years). If you are adding new files, you need to use the
    target checks for additional warnings. Your patches shouldn't introduce any
    new warning (but it may be possible to remove some warning checks if it makes
    the code much easier). You can also use `cmake --preset develop` if you prefer.
-5. Test your changes by running `./run-tests.sh` in the `test` directory.
+5. Test your changes by running `./run-tests.sh` in the `test` directory, or using `ctest`.
    (You must run `./fetch-test-deps.sh` first; if you forget to, the test suite
    will fail and remind you mid-way.)
 6. Format your changes according to `clang-format`, which will reformat the

test/CMakeLists.txt

@@ -1,14 +1,13 @@
 # SPDX-License-Identifier: MIT
 
 option(TESTS_RUN_NONFREE "Run tests that build nonfree codebases." ON)
-option(TESTS_RUN_EXTERNAL "Run tests that build external codebases." ON)
 set(TESTS_OS_NAME "" CACHE STRING "Skip running tests known to fail on this OS.")
 
 add_executable(randtilegen gfx/randtilegen.cpp)
 add_executable(rgbgfx_test gfx/rgbgfx_test.cpp)
 set_target_properties(randtilegen rgbgfx_test PROPERTIES
 # hack for MSVC: no-op generator expression to stop generation of "per-configuration subdirectory"
-                      RUNTIME_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/gfx>)
+                      RUNTIME_OUTPUT_DIRECTORY "$<1:${CMAKE_CURRENT_SOURCE_DIR}/gfx>")
 foreach(prog "randtilegen" "rgbgfx_test")
   target_link_libraries(${prog} PRIVATE PNG::PNG)
   # Copy the DLLs in the output directory so the program can be run for testing without having to `install`.
@@ -24,18 +23,30 @@ if(NOT TESTS_RUN_NONFREE)
   set(ONLY_FREE "--only-free")
 endif()
 
-set(ONLY_INTERNAL)
-if(NOT TESTS_RUN_EXTERNAL)
-  set(ONLY_INTERNAL "--only-internal")
-endif()
-
 set(OS_NAME)
 if(NOT TESTS_OS_NAME STREQUAL "")
   set(OS_NAME "--os" "${TESTS_OS_NAME}")
 endif()
 
-configure_file(CTestCustom.cmake.in ${CMAKE_BINARY_DIR}/CTestCustom.cmake @ONLY)
+foreach(component "asm" "link" "fix" "gfx")
+  add_test(NAME "rgb${component}"
+           COMMAND bash -- test.sh
+           WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${component}")
+  set_tests_properties("rgb${component}" PROPERTIES REQUIRED_FILES "$<TARGET_FILE:rgb${component}>"
+                                                    PROCESSORS 1
+                                                    LABELS "rgb${component}")
+endforeach()
+set_tests_properties(rgbgfx PROPERTIES REQUIRED_FILES "$<TARGET_FILE:rgbgfx>;$<TARGET_FILE:randtilegen>;$<TARGET_FILE:rgbgfx_test>")
 
-add_test(NAME all
-         COMMAND ./run-tests.sh ${ONLY_FREE} ${ONLY_INTERNAL} ${OS_NAME}
-         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_test(NAME fetch-test-deps
+         COMMAND bash -- fetch-test-deps.sh ${ONLY_FREE}
+         WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+set_tests_properties(fetch-test-deps PROPERTIES FIXTURES_SETUP "external-repos"
+                                                LABELS "external")
+add_test(NAME external
+         COMMAND bash -- run-tests.sh --only-external ${ONLY_FREE} ${OS_NAME}
+         WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+set_tests_properties(external PROPERTIES DEPENDS "rgbasm;rgblink;rgbfix;rgbgfx" # Only attempt building whole projects if each tool passes muster on its own.
+                                         PROCESSORS 4
+                                         FIXTURES_REQUIRED "external-repos"
+                                         LABELS "external") # Allow filtering out external tests.

test/CTestCustom.cmake.in

@@ -1 +0,0 @@
-set(CTEST_CUSTOM_PRE_TEST "@CMAKE_CURRENT_SOURCE_DIR@/fetch-test-deps.sh @ONLY_FREE@ @ONLY_INTERNAL@")