From 6b0658fb5912e50ca32e1c17e0c43e99a6777f0f Mon Sep 17 00:00:00 2001 From: ISSOtm Date: Thu, 19 Mar 2026 02:51:51 +0100 Subject: [PATCH] Cache library deps on Windows Besides the immediate performance improvement, reducing the frequency of downloads should make spurious failures (network, rate-limiting...) less bothersome. --- .github/scripts/get_win_deps.ps1 | 18 ----------- .github/scripts/install_deps.sh | 8 ++++- .../workflows/create-release-artifacts.yml | 18 ++++++++--- .github/workflows/testing.yml | 31 ++++++++++++------- ARCHITECTURE.md | 6 +++- CMakeLists.txt | 19 ++---------- cmake/deps.cmake | 23 ++++++++++++++ 7 files changed, 70 insertions(+), 53 deletions(-) delete mode 100644 .github/scripts/get_win_deps.ps1 create mode 100644 cmake/deps.cmake diff --git a/.github/scripts/get_win_deps.ps1 b/.github/scripts/get_win_deps.ps1 deleted file mode 100644 index 57c31423..00000000 --- a/.github/scripts/get_win_deps.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -function getlibrary ([string] $URI, [string] $filename, [string] $hash, [string] $destdir) { - $wc = New-Object Net.WebClient - [string] $downloadhash = $null - try { - $wc.DownloadFile($URI, $filename) - $downloadhash = $(Get-FileHash $filename -Algorithm SHA256).Hash - } catch { - Write-Host "${filename}: failed to download" - exit 1 - } - if ($hash -ne $downloadhash) { - Write-Host "${filename}: SHA256 mismatch ($downloadhash)" - exit 1 - } - Expand-Archive -DestinationPath $destdir $filename -} - -getlibrary 'https://github.com/lexxmark/winflexbison/releases/download/v2.5.25/win_flex_bison-2.5.25.zip' 'winflexbison.zip' '8d324b62be33604b2c45ad1dd34ab93d722534448f55a16ca7292de32b6ac135' bison diff --git a/.github/scripts/install_deps.sh b/.github/scripts/install_deps.sh index f417b2e7..420442b3 100755 --- a/.github/scripts/install_deps.sh +++ b/.github/scripts/install_deps.sh @@ -7,7 +7,7 @@ case "${1%-*}" in sudo apt-get -qq update sudo apt-get install -yq bison libpng-dev pkg-config ;; - macos) + macos) # macOS bundles GNU Make 3.81, which doesn't support synced output. # We leave it as the default in `PATH`, to test that our Makefile works with it. # However, CMake automatically uses Homebrew's `gmake`, so our CI has synced output. @@ -20,11 +20,17 @@ case "${1%-*}" in freebsd) pkg install -y bash bison cmake git png ;; + windows) + choco install -y winflexbison3 + # The below expects the base name, not the Windows-specific name. + bison() { win_bison "$@"; } # An alias doesn't work, so we use a function instead. + ;; *) echo "WARNING: Cannot install deps for OS '$1'" ;; esac +echo "PATH=($PATH)" | sed 's/:/\n /g' bison --version make --version cmake --version diff --git a/.github/workflows/create-release-artifacts.yml b/.github/workflows/create-release-artifacts.yml index 74227618..0192be7f 100644 --- a/.github/workflows/create-release-artifacts.yml +++ b/.github/workflows/create-release-artifacts.yml @@ -14,6 +14,10 @@ env: CMAKE_BUILD_TYPE: Release # `cmake -S` now implies `-DCMAKE_BUILD_TYPE=Release`. CMAKE_CONFIG_TYPE: Release # `cmake --build` now implies `--config Release`. + # We instruct CMake to download and build third-party projects in the same place as in `testing.yml` + # for the sources cache to remain valid. + DEPS_ROOT_DIR: ~/_deps # Note that this needs to be used in a position where Bash will trigger tilde expansion! + jobs: windows: runs-on: windows-2022 @@ -35,17 +39,21 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Install deps - run: .github/scripts/get_win_deps.ps1 + run: .github/scripts/install_deps.sh windows + - name: Cache library deps + uses: actions/cache@v5 + with: + path: ${{ env.DEPS_ROOT_DIR }}/*-tmp/ + key: dep-srcs-${{ hashFiles('cmake/deps.cmake') }} + enableCrossOsArchive: true - name: Build Windows binaries shell: bash - # We instruct CMake to download and build third-party projects outside of our source tree, - # otherwise they can trigger `-Werror=dev` (from the `develop` preset). run: | - cmake -S . -B build --preset msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="$TEMP/cmake-deps" -DBISON_EXECUTABLE=bison/win_bison.exe + cmake -S . -B build --preset msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}" cmake --build build - cmake --install build --config Release --prefix install_dir --verbose - name: Package binaries run: | + cmake --install build --config Release --prefix install_dir --verbose Compress-Archive -LiteralPath @("install_dir/bin/rgbasm.exe", "install_dir/bin/rgblink.exe", "install_dir/bin/rgbfix.exe", "install_dir/bin/rgbgfx.exe", "install_dir/bin/z.dll", "install_dir/bin/libpng16.dll") "rgbds-win${{ matrix.bits }}.zip" - name: Upload Windows binaries uses: actions/upload-artifact@v4 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c4694b5e..a59854d4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -12,6 +12,10 @@ env: CMAKE_INSTALL_PARALLEL_LEVEL: 4 # `cmake --install` now implies `--parallel 4`. CMAKE_CONFIG_TYPE: Debug # `cmake --build` now implies `--config Debug`. + # We instruct CMake to download and build third-party projects outside of our source tree, + # otherwise they can trigger `-Werror=dev` (from the `develop` preset). + DEPS_ROOT_DIR: ~/_deps # Note that this needs to be used in a position where Bash will trigger tilde expansion! + jobs: unix: strategy: @@ -20,9 +24,7 @@ jobs: cxx: [g++, clang++] buildsys: [make, cmake] exclude: - # Don't use `g++` on macOS; it's just an alias to `clang++`. - - os: macos-14 - cxx: g++ + - { os: macos-14, cxx: g++ } # Don't use `g++` on macOS; it's just an alias to `clang++`. fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -67,7 +69,7 @@ jobs: tee -a <<<"hash=${hash%-}" $GITHUB_OUTPUT - name: Check test dependency repositories cache id: test-deps-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ fromJSON(steps.test-deps-cache-params.outputs.paths) }} key: ${{ matrix.os }}-${{ steps.test-deps-cache-params.outputs.hash }} @@ -124,7 +126,7 @@ jobs: tee -a <<<"hash=${hash%-}" $GITHUB_OUTPUT - name: Check test dependency repositories cache id: test-deps-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ fromJSON(steps.test-deps-cache-params.outputs.paths) }} key: ${{ matrix.os }}-${{ steps.test-deps-cache-params.outputs.hash }} @@ -158,18 +160,23 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - name: Install deps - run: .github/scripts/get_win_deps.ps1 + run: | + bash .github/scripts/install_deps.sh ${{ matrix.os }} + - name: Cache library deps + uses: actions/cache@v5 + with: + path: ${{ env.DEPS_ROOT_DIR }}/*-tmp/ + key: dep-srcs-${{ hashFiles('cmake/deps.cmake') }} + enableCrossOsArchive: true - name: Build Windows binaries shell: bash - # We instruct CMake to download and build third-party projects outside of our source tree, - # otherwise they can trigger `-Werror=dev` (from the `develop` preset). run: | # ASan seems to be broken on Windows, so we disable it. - cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="$TEMP/cmake-deps" -DBISON_EXECUTABLE=bison/win_bison.exe -DSANITIZERS=OFF + cmake -S . -B build --preset develop-msvc${{ matrix.bits }} -DFETCHCONTENT_BASE_DIR="${{ env.DEPS_ROOT_DIR }}" -DSANITIZERS=OFF cmake --build build - cmake --install build --config Debug --prefix install_dir --verbose - name: Package binaries shell: bash run: | + cmake --install build --config Debug --prefix install_dir --verbose mkdir bins cp -v install_dir/bin/{rgb*.exe,*.dll} bins - name: Upload Windows binaries @@ -187,7 +194,7 @@ jobs: tee -a <<<"hash=${hash%-}" $GITHUB_OUTPUT - name: Check test dependency repositories cache id: test-deps-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ fromJSON(steps.test-deps-cache-params.outputs.paths) }} key: ${{ matrix.os }}-${{ matrix.bits }}-${{ steps.test-deps-cache-params.outputs.hash }} @@ -296,7 +303,7 @@ jobs: tee -a <<<"hash=${hash%-}" $GITHUB_OUTPUT - name: Check test dependency repositories cache id: test-deps-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ fromJSON(steps.test-deps-cache-params.outputs.paths) }} key: mingw-${{ matrix.bits }}-${{ steps.test-deps-cache-params.outputs.hash }} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 09326bbe..cffcf179 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -28,6 +28,8 @@ rgbds/ │ │ └── ... │ └── workflows/ │ └── ... +├── cmake/ +│ └── ... ├── contrib/ │ ├── bash_compl/ │ ├── zsh_compl/ @@ -70,6 +72,8 @@ rgbds/ Scripts used by GitHub Actions workflow files. * **`workflows/`:** GitHub Actions CI workflow description files. Used for automated testing, deployment, etc. +- **`cmake/`**: + Files relevant to our CMake build system that are not required to be somewhere else (e.g. `CMakePresets.json` *has* to be at the root). - **`contrib/`:** Scripts and other resources which may be useful to RGBDS users and developers. * **`bash_compl/`:** @@ -108,7 +112,7 @@ rgbds/ Configuration for C++ static analysis with [`clang-tidy`](https://clang.llvm.org/extra/clang-tidy/) (for which we define the shortcut `make tidy`). - **`CMakeLists.txt`:** Defines how to build RGBDS with CMake. -- **`CMakePresets.json`:** +- **`CMakePresets.json`:** Defines some [presets](https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html) useful for working with our CMake. - **`compile_flags.txt`:** Compiler flags for `clang-tidy`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 7866b60b..a5e68d85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,26 +140,13 @@ endif() ## Dependencies. include(FetchContent) -FetchContent_Declare(PNG - URL https://download.sourceforge.net/libpng/libpng-1.6.55.tar.xz - URL_HASH SHA256=d925722864837ad5ae2a82070d4b2e0603dc72af44bd457c3962298258b8e82d - FIND_PACKAGE_ARGS 1.5.4) -FetchContent_Declare(ZLIB - URL https://www.zlib.net/zlib-1.3.2.tar.xz - URL_HASH SHA256=d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 -# libpng documents requiring "zlib 1.0.4 or later (1.2.13 or later recommended for performance and security reasons)". -# We thus enforce 1.0.4, but note that the libpng source code mentions that "it may work with versions as old as zlib 0.95". - FIND_PACKAGE_ARGS 1.0.4) -set(ZLIB_BUILD_SHARED ON CACHE INTERNAL "") -set(ZLIB_BUILD_STATIC OFF CACHE INTERNAL "") +include(cmake/deps.cmake) + FetchContent_MakeAvailable(ZLIB) if(NOT DEFINED ZLIB_INCLUDE_DIRS) set(ZLIB_INCLUDE_DIRS "${zlib_BINARY_DIR};${zlib_SOURCE_DIR}") # libpng's `genout` script relies on this variable to be set. endif() -set(PNG_TESTS OFF CACHE INTERNAL "") # We do not care for these two (and they can even cause compile errors!) -set(PNG_TOOLS OFF CACHE INTERNAL "") -set(PNG_SHARED ON CACHE INTERNAL "") # Upstream seems to favour the dynamic lib over the static one? -set(PNG_STATIC OFF CACHE INTERNAL "") + FetchContent_MakeAvailable(PNG) if(NOT TARGET PNG::PNG) if(PNG_SHARED) diff --git a/cmake/deps.cmake b/cmake/deps.cmake new file mode 100644 index 00000000..81ece158 --- /dev/null +++ b/cmake/deps.cmake @@ -0,0 +1,23 @@ +# This file declares the dependencies we use, using `FetchContent`. +# https://cmake.org/cmake.help/latest/guide/using-dependencies/index.html#downloading-and-building-from-source-with-fetchcontent +# These are kept in a separate file so that it can be hashed as a key for our CI's `actions/cache`. + +FetchContent_Declare(PNG + URL https://download.sourceforge.net/libpng/libpng-1.6.55.tar.xz + URL_HASH SHA256=d925722864837ad5ae2a82070d4b2e0603dc72af44bd457c3962298258b8e82d + FIND_PACKAGE_ARGS 1.5.4) + +set(PNG_TESTS OFF CACHE INTERNAL "") # We do not care for these two (and they can even cause compile errors!) +set(PNG_TOOLS OFF CACHE INTERNAL "") +set(PNG_SHARED ON CACHE INTERNAL "") # Upstream seems to favour the dynamic lib over the static one? +set(PNG_STATIC OFF CACHE INTERNAL "") + +FetchContent_Declare(ZLIB + URL https://www.zlib.net/zlib-1.3.2.tar.xz + URL_HASH SHA256=d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 +# libpng documents requiring "zlib 1.0.4 or later (1.2.13 or later recommended for performance and security reasons)". +# We thus enforce 1.0.4, but note that the libpng source code mentions that "it may work with versions as old as zlib 0.95". + FIND_PACKAGE_ARGS 1.0.4) + +set(ZLIB_BUILD_SHARED ON CACHE INTERNAL "") +set(ZLIB_BUILD_STATIC OFF CACHE INTERNAL "")