From 78281a4aaa99b0d5d482a4a48c2e7d307ef1364c Mon Sep 17 00:00:00 2001 From: Eldred Habert Date: Mon, 13 Apr 2026 03:29:12 +0200 Subject: [PATCH] Break down CTest cases into each test suite (#1931) This allows parallelising the individual programs' test suites, which can save a few minutes! --- .github/workflows/testing.yml | 24 ++++++++++++++++-------- CMakeLists.txt | 6 ++++++ docs/CONTRIBUTING.md | 2 +- test/CMakeLists.txt | 33 ++++++++++++++++++++++----------- test/CTestCustom.cmake.in | 1 - 5 files changed, 45 insertions(+), 21 deletions(-) delete mode 100644 test/CTestCustom.cmake.in diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 96c1dc7b..a51eafd4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,11 +6,14 @@ on: env: CLICOLOR_FORCE: 1 # Tells CMake to have colored output. CMAKE_COLOR_DIAGNOSTICS: ON # Tells CMake-generated build systems to have colored output. + CMAKE_CONFIG_TYPE: Debug # `cmake --build` now implies `--config Debug`. # Approximate number of CPU cores in GitHub's runners as of 2026-03-18: # https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories CMAKE_BUILD_PARALLEL_LEVEL: 4 # `cmake --build` now implies `--parallel 4`. CMAKE_INSTALL_PARALLEL_LEVEL: 4 # `cmake --install` now implies `--parallel 4`. - CMAKE_CONFIG_TYPE: Debug # `cmake --build` now implies `--config Debug`. + CTEST_PARALLEL_LEVEL: 0 # `ctest` now implies `--parallel 0` (number of logical CPUs). + CTEST_NO_TESTS_ACTION: error # Make CTest fail if it cannot find any tests. (That should never happen.) + CTEST_OUTPUT_ON_FAILURE: ON # CTest reports test program output on failure. GIT_CONFIG_COUNT: 1 GIT_CONFIG_KEY_0: color.ui @@ -96,7 +99,7 @@ jobs: - name: Run tests using CTest if: matrix.buildsys == 'cmake' run: | - ctest --test-dir build --verbose + ctest --test-dir build --schedule-random macos-static: runs-on: macos-14 @@ -168,7 +171,8 @@ jobs: enableCrossOsArchive: true # Currently only used on Windows, but the contents are OS-agnostic. - name: Build Windows binaries run: | # ASan seems to be broken on Windows, so we disable it. - cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}" -DSANITIZERS=OFF + cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DSANITIZERS=OFF \ + -DTESTS_OS_NAME=${{ matrix.os }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}" cmake --build build - name: Package binaries working-directory: build @@ -201,9 +205,9 @@ jobs: - name: Install test dependency dependencies run: | test/fetch-test-deps.sh --get-deps ${{ matrix.os }} - - name: Run tests + - name: Run tests using CTest run: | - test/run-tests.sh --os ${{ matrix.os }} + ctest --test-dir build --schedule-random -C Debug windows-mingw-build: strategy: @@ -353,13 +357,17 @@ jobs: - name: Build & test using CMake on FreeBSD uses: vmactions/freebsd-vm@v1 with: - envs: CLICOLOR_FORCE CMAKE_COLOR_DIAGNOSTICS CMAKE_BUILD_PARALLEL_LEVEL CMAKE_INSTALL_PARALLEL_LEVEL + envs: >- + CLICOLOR_FORCE CMAKE_COLOR_DIAGNOSTICS CMAKE_CONFIG_TYPE + CMAKE_BUILD_PARALLEL_LEVEL CMAKE_INSTALL_PARALLEL_LEVEL CTEST_PARALLEL_LEVEL + CTEST_NO_TESTS_ACTION CTEST_OUTPUT_ON_FAILURE + GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0 release: "14.3" usesh: true prepare: | .github/scripts/install_deps.sh freebsd run: | # Leak detection is not supported on FreeBSD, so disable it. - cmake -S . -B build --preset develop -DTESTS_RUN_EXTERNAL=OFF -DTESTS_OS_NAME=freebsd + cmake -S . -B build --preset develop -DTESTS_OS_NAME=freebsd cmake --build build --verbose + ASAN_OPTIONS=detect_leaks=0 ctest --test-dir build --schedule-random --label-exclude external cmake --install build --verbose - ASAN_OPTIONS=detect_leaks=0 ctest --test-dir build --verbose diff --git a/CMakeLists.txt b/CMakeLists.txt index a1b0d907..320a76c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,11 +136,17 @@ endif() include(FetchContent) include(cmake/deps.cmake) +if(NOT DEFINED ZLIB_BUILD_TESTING) # Unless overridden (e.g. in the cache), we don't care about zlib's tests. + set(ZLIB_BUILD_TESTING OFF) +endif() FetchContent_MakeAvailable(ZLIB) if(NOT DEFINED ZLIB_INCLUDE_DIRS) set(ZLIB_INCLUDE_DIRS "${zlib_BINARY_DIR};${zlib_SOURCE_DIR}") # libpng's `genout` script relies on this variable to be set. endif() +if(NOT DEFINED PNG_TESTS) # Unless overridden (e.g. in the cache), we don't care about libpng's tests. + set(PNG_TESTS OFF) +endif() FetchContent_MakeAvailable(PNG) if(NOT TARGET PNG::PNG) if(PNG_SHARED) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 1ffbacc4..6adf5855 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -65,7 +65,7 @@ years). If you are adding new files, you need to use the target checks for additional warnings. Your patches shouldn't introduce any new warning (but it may be possible to remove some warning checks if it makes the code much easier). You can also use `cmake --preset develop` if you prefer. -5. Test your changes by running `./run-tests.sh` in the `test` directory. +5. Test your changes by running `./run-tests.sh` in the `test` directory, or using `ctest`. (You must run `./fetch-test-deps.sh` first; if you forget to, the test suite will fail and remind you mid-way.) 6. Format your changes according to `clang-format`, which will reformat the diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fbd3de28..4a305a72 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,14 +1,13 @@ # SPDX-License-Identifier: MIT option(TESTS_RUN_NONFREE "Run tests that build nonfree codebases." ON) -option(TESTS_RUN_EXTERNAL "Run tests that build external codebases." ON) set(TESTS_OS_NAME "" CACHE STRING "Skip running tests known to fail on this OS.") add_executable(randtilegen gfx/randtilegen.cpp) add_executable(rgbgfx_test gfx/rgbgfx_test.cpp) set_target_properties(randtilegen rgbgfx_test PROPERTIES # hack for MSVC: no-op generator expression to stop generation of "per-configuration subdirectory" - RUNTIME_OUTPUT_DIRECTORY $<1:${CMAKE_CURRENT_SOURCE_DIR}/gfx>) + RUNTIME_OUTPUT_DIRECTORY "$<1:${CMAKE_CURRENT_SOURCE_DIR}/gfx>") foreach(prog "randtilegen" "rgbgfx_test") target_link_libraries(${prog} PRIVATE PNG::PNG) # Copy the DLLs in the output directory so the program can be run for testing without having to `install`. @@ -24,18 +23,30 @@ if(NOT TESTS_RUN_NONFREE) set(ONLY_FREE "--only-free") endif() -set(ONLY_INTERNAL) -if(NOT TESTS_RUN_EXTERNAL) - set(ONLY_INTERNAL "--only-internal") -endif() - set(OS_NAME) if(NOT TESTS_OS_NAME STREQUAL "") set(OS_NAME "--os" "${TESTS_OS_NAME}") endif() -configure_file(CTestCustom.cmake.in ${CMAKE_BINARY_DIR}/CTestCustom.cmake @ONLY) +foreach(component "asm" "link" "fix" "gfx") + add_test(NAME "rgb${component}" + COMMAND bash -- test.sh + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${component}") + set_tests_properties("rgb${component}" PROPERTIES REQUIRED_FILES "$" + PROCESSORS 1 + LABELS "rgb${component}") +endforeach() +set_tests_properties(rgbgfx PROPERTIES REQUIRED_FILES "$;$;$") -add_test(NAME all - COMMAND ./run-tests.sh ${ONLY_FREE} ${ONLY_INTERNAL} ${OS_NAME} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +add_test(NAME fetch-test-deps + COMMAND bash -- fetch-test-deps.sh ${ONLY_FREE} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") +set_tests_properties(fetch-test-deps PROPERTIES FIXTURES_SETUP "external-repos" + LABELS "external") +add_test(NAME external + COMMAND bash -- run-tests.sh --only-external ${ONLY_FREE} ${OS_NAME} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") +set_tests_properties(external PROPERTIES DEPENDS "rgbasm;rgblink;rgbfix;rgbgfx" # Only attempt building whole projects if each tool passes muster on its own. + PROCESSORS 4 + FIXTURES_REQUIRED "external-repos" + LABELS "external") # Allow filtering out external tests. diff --git a/test/CTestCustom.cmake.in b/test/CTestCustom.cmake.in deleted file mode 100644 index 5f1f0869..00000000 --- a/test/CTestCustom.cmake.in +++ /dev/null @@ -1 +0,0 @@ -set(CTEST_CUSTOM_PRE_TEST "@CMAKE_CURRENT_SOURCE_DIR@/fetch-test-deps.sh @ONLY_FREE@ @ONLY_INTERNAL@")