From ff26b00cb1363c3deda67cf5a580d8d7adb7cde8 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Thu, 18 Dec 2025 12:15:26 -0800 Subject: [PATCH] new[CI]: add linux sanitizer tests (#2860) Signed-off-by: Melissa Kilby --- .github/scripts/build-sanitizer-tests.sh | 48 ++++++++++++++++++ .github/workflows/build_and_test.yml | 33 +++++++++++++ CMakeLists.txt | 62 ++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + 4 files changed, 145 insertions(+) create mode 100755 .github/scripts/build-sanitizer-tests.sh diff --git a/.github/scripts/build-sanitizer-tests.sh b/.github/scripts/build-sanitizer-tests.sh new file mode 100755 index 00000000..9c911890 --- /dev/null +++ b/.github/scripts/build-sanitizer-tests.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -ex + +export CMAKE_C_COMPILER=/usr/bin/clang +export CMAKE_CXX_COMPILER=/usr/bin/clang++ +BASE_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_COMPILE_WARNING_AS_ERROR=ON" +if [[ "$(uname -s)" != "Darwin" ]]; then + BASE_CMAKE_ARGS+=" -DMLX_BUILD_METAL=OFF" +fi + +run_test() { + local sanitizer_name=$1 + local cmake_sanitizer_flag="-DUSE_${sanitizer_name}=ON" + echo " Running tests with: ${sanitizer_name}" + + case "$sanitizer_name" in + ASAN) + export ASAN_OPTIONS="detect_leaks=0" + ;; + UBSAN) + export UBSAN_OPTIONS="halt_on_error=0:print_stacktrace=1" + ;; + TSAN) + export TSAN_OPTIONS="" + ;; + esac + + rm -rf build + mkdir -p build + pushd build > /dev/null + + cmake .. ${BASE_CMAKE_ARGS} ${cmake_sanitizer_flag} + make -j $(nproc) + ./tests/tests + + popd > /dev/null + unset ${sanitizer_name}_OPTIONS +} + +sanitizer_arg=$(echo "$1" | tr '[:lower:]' '[:upper:]') + +if [[ "$sanitizer_arg" == "ASAN" || "$sanitizer_arg" == "UBSAN" || "$sanitizer_arg" == "TSAN" ]]; then + run_test "$sanitizer_arg" + echo " ${sanitizer_arg} test run completed successfully." +else + echo "Error: Invalid sanitizer '$1'. Please use one of: ASAN, UBSAN, TSAN." + exit 1 +fi diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 34ff5505..c83d495b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -84,6 +84,39 @@ jobs: - uses: actions/checkout@v6 - uses: ./.github/actions/build-docs + linux_sanitizer_build_and_test: + name: Linux Sanitizer Tests (${{ matrix.sanitizer }}) + needs: check_lint + strategy: + fail-fast: false + matrix: + sanitizer: [ASAN, UBSAN] + # todo 12/16/2025: enable TSAN later + consider enabling ASAN for GPU backend tests. + # sanitizer: [ASAN, UBSAN, TSAN] + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install Dependencies + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update -y + sudo apt-get install -y \ + build-essential \ + libblas-dev \ + liblapacke-dev \ + libopenblas-dev \ + cmake \ + clang \ + git + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + + - name: Linux Build and Test with ${{ matrix.sanitizer }} + run: | + bash .github/scripts/build-sanitizer-tests.sh ${{ matrix.sanitizer }} + linux_fedora_build_cpp: name: Linux Fedora (${{ matrix.arch }}) needs: check_lint diff --git a/CMakeLists.txt b/CMakeLists.txt index 2451aa65..705e15a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,9 @@ option(MLX_METAL_JIT "Use JIT compilation for Metal kernels" OFF) option(MLX_USE_CCACHE "Use CCache for compilation cache when available" ON) option(BUILD_SHARED_LIBS "Build mlx as a shared library" OFF) option(USE_SYSTEM_FMT "Use system's provided fmt library" OFF) +option(USE_ASAN "Enable AddressSanitizer (ASan)" OFF) +option(USE_UBSAN "Enable UndefinedBehaviorSanitizer (UBSan)" OFF) +option(USE_TSAN "Enable ThreadSanitizer (TSan)" OFF) # --------------------- Processor tests ------------------------- message( @@ -81,6 +84,63 @@ if(MLX_USE_CCACHE) endif() endif() +if(USE_ASAN AND USE_TSAN) + message( + FATAL_ERROR + "AddressSanitizer (ASan) and ThreadSanitizer (TSan) are mutually exclusive and cannot be enabled at the same time." + ) +endif() + +set(SANITIZER_COMPILE_FLAGS "") +set(SANITIZER_LINK_FLAGS "") + +if(USE_ASAN) + if(WIN32 AND MSVC) + list(APPEND SANITIZER_COMPILE_FLAGS /fsanitize=address) + list(APPEND SANITIZER_LINK_FLAGS /fsanitize=address) + else() + list(APPEND SANITIZER_COMPILE_FLAGS -fsanitize=address) + list(APPEND SANITIZER_LINK_FLAGS -fsanitize=address) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND SANITIZER_LINK_FLAGS -lpthread) + endif() + endif() +endif() + +if(USE_UBSAN) + if(WIN32 AND MSVC) + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + list(APPEND SANITIZER_COMPILE_FLAGS -fsanitize=undefined) + list(APPEND SANITIZER_LINK_FLAGS -fsanitize=undefined) + else() + message( + WARNING + "UndefinedBehaviorSanitizer (UBSan) is not directly supported via a simple flag in MSVC." + ) + endif() + else() + list(APPEND SANITIZER_COMPILE_FLAGS -fsanitize=undefined) + list(APPEND SANITIZER_LINK_FLAGS -fsanitize=undefined) + endif() +endif() + +if(USE_TSAN) + if(WIN32 AND MSVC) + message( + FATAL_ERROR + "ThreadSanitizer (TSan) is not supported by the MSVC compiler. Please use Clang or GCC." + ) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + message(FATAL_ERROR "ThreadSanitizer (TSan) is not supported on macOS.") + else() + list(APPEND SANITIZER_COMPILE_FLAGS -fsanitize=thread) + list(APPEND SANITIZER_LINK_FLAGS -fsanitize=thread) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND SANITIZER_LINK_FLAGS -lpthread) + endif() + endif() +endif() + # ----------------------------- Lib ----------------------------- include(FetchContent) @@ -93,6 +153,8 @@ add_library(mlx) # ‘std::pair’ when C++17 is enabled changed to match C++14 in GCC # 10.1 target_compile_options(mlx PRIVATE -Wno-psabi) +target_compile_options(mlx PUBLIC ${SANITIZER_COMPILE_FLAGS}) +target_link_options(mlx PUBLIC ${SANITIZER_LINK_FLAGS}) if(MLX_BUILD_CUDA) enable_language(CUDA) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eafa9889..7cac06c9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -37,5 +37,7 @@ target_sources( ${METAL_TEST_SOURCES}) target_link_libraries(tests PRIVATE mlx doctest) +target_compile_options(tests PRIVATE ${SANITIZER_COMPILE_FLAGS}) +target_link_options(tests PRIVATE ${SANITIZER_LINK_FLAGS}) doctest_discover_tests(tests) add_test(NAME tests COMMAND tests)