From c6fc05da74115cad67e0c8694d54419003c923d4 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 14 Jan 2026 05:56:50 -0500 Subject: [PATCH 1/4] Enable ASAN for mpi4py in CI Run mpi4py with ASAN, with a separate step that aborts on errors. The existing steps should run to completion even if an error is detected. Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py.yaml | 54 +++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ompi_mpi4py.yaml b/.github/workflows/ompi_mpi4py.yaml index 29abbcaf70d..dd8cdaa68df 100644 --- a/.github/workflows/ompi_mpi4py.yaml +++ b/.github/workflows/ompi_mpi4py.yaml @@ -20,18 +20,33 @@ permissions: jobs: test: - runs-on: ubuntu-22.04 + # We need Unbuntu 24.04 (over 22.04) due to a kernel bug, + # see https://github.com/google/sanitizers/issues/856. + runs-on: ubuntu-24.04 timeout-minutes: 30 env: MPI4PY_TEST_SPAWN: true + # disable ODR violation detection until #13469 is fixed + # and don't abort on error by default + ASAN_OPTIONS: verify_asan_link_order=0,detect_odr_violation=0,abort_on_error=0 + # disable leak detection and make sure we do not fail on leaks + LSAN_OPTIONS: detect_leaks=0,exitcode=0 + steps: - name: Configure hostname run: echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + - name: Print kernel version + run: uname -a + + - name: Disable ASLR + run: sudo sysctl -w kernel.randomize_va_space=0 + if: ${{ runner.os == 'Linux' }} + - name: Install depencencies - run: sudo apt-get install -y -q - libnuma-dev + run: sudo apt update && sudo apt-get install -y -q + libnuma-dev libasan8 if: ${{ runner.os == 'Linux' }} - name: Checkout Open MPI @@ -59,7 +74,8 @@ jobs: --disable-oshmem --disable-silent-rules --prefix=/opt/openmpi - LDFLAGS=-Wl,-rpath,/opt/openmpi/lib + CFLAGS="-fno-omit-frame-pointer -g -O1 -fsanitize=address" + LDFLAGS="-Wl,-rpath,/opt/openmpi/lib -fsanitize=address" working-directory: mpi-build - name: Build MPI @@ -145,6 +161,35 @@ jobs: if: ${{ true }} timeout-minutes: 10 + - name: Setting up ASAN environment + # LD_PRELOAD is needed to make sure ASAN is the first thing loaded + # as it will otherwise complain + # Leak detection is currently disabled because of the size of the report. + # The patcher is disabled because ASAN fails if code mmaps data at fixed + # memory addresses, see https://github.com/open-mpi/ompi/issues/12819 + # ODR violation detection is disabled until #13469 is fixed + run: | + echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV + echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV + + - name: Test mpi4py ASAN (np=1) + run: mpiexec -n 1 python test/main.py -v -x TestExcErrhandlerNull + if: ${{ true }} + timeout-minutes: 10 + + - name: Test mpi4py ASAN (np=4) + run: mpiexec -n 4 python test/main.py -v -f -x TestExcErrhandlerNull + if: ${{ true }} + timeout-minutes: 10 + + - name: Disabling ASAN environment + run: | + echo LD_PRELOAD= >> $GITHUB_ENV + echo ASAN_OPTIONS=verify_asan_link_order=0,detect_odr_violation=0,abort_on_error=0 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV + - name: Relocate Open MPI installation run: mv /opt/openmpi /opt/ompi - name: Update PATH and set OPAL_PREFIX and LD_LIBRARY_PATH @@ -157,4 +202,3 @@ jobs: run: python test/main.py -v -x TestExcErrhandlerNull if: ${{ true }} timeout-minutes: 10 - From 7f5eea7572f444906b8919a23bbd1de9db730abc Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 14 Jan 2026 16:31:25 -0500 Subject: [PATCH 2/4] Remove disable of ASLR and apt update Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ompi_mpi4py.yaml b/.github/workflows/ompi_mpi4py.yaml index dd8cdaa68df..3f8b57d4736 100644 --- a/.github/workflows/ompi_mpi4py.yaml +++ b/.github/workflows/ompi_mpi4py.yaml @@ -37,15 +37,15 @@ jobs: run: echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} - - name: Print kernel version - run: uname -a - - - name: Disable ASLR - run: sudo sysctl -w kernel.randomize_va_space=0 - if: ${{ runner.os == 'Linux' }} +# - name: Print kernel version +# run: uname -a +# +# - name: Disable ASLR +# run: sudo sysctl -w kernel.randomize_va_space=0 +# if: ${{ runner.os == 'Linux' }} - name: Install depencencies - run: sudo apt update && sudo apt-get install -y -q + run: sudo apt-get install -y -q libnuma-dev libasan8 if: ${{ runner.os == 'Linux' }} From 1af95bb437e4dc7807768019294c7913654e6a64 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 14 Jan 2026 17:04:31 -0500 Subject: [PATCH 3/4] Enable ASAN for all mpi4py tests and increase optimizations 30 minutes are not enough to run two extra tests so just enable ASAN for the existing tests. Also test `ompi_info` and `mpicc`. Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py.yaml | 52 ++++++++++++------------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ompi_mpi4py.yaml b/.github/workflows/ompi_mpi4py.yaml index 3f8b57d4736..0e5939118d2 100644 --- a/.github/workflows/ompi_mpi4py.yaml +++ b/.github/workflows/ompi_mpi4py.yaml @@ -26,10 +26,9 @@ jobs: timeout-minutes: 30 env: MPI4PY_TEST_SPAWN: true - # disable ODR violation detection until #13469 is fixed - # and don't abort on error by default + # disable ASAN while building ASAN_OPTIONS: verify_asan_link_order=0,detect_odr_violation=0,abort_on_error=0 - # disable leak detection and make sure we do not fail on leaks + # disable leak detection LSAN_OPTIONS: detect_leaks=0,exitcode=0 steps: @@ -37,13 +36,6 @@ jobs: run: echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} -# - name: Print kernel version -# run: uname -a -# -# - name: Disable ASLR -# run: sudo sysctl -w kernel.randomize_va_space=0 -# if: ${{ runner.os == 'Linux' }} - - name: Install depencencies run: sudo apt-get install -y -q libnuma-dev libasan8 @@ -74,7 +66,7 @@ jobs: --disable-oshmem --disable-silent-rules --prefix=/opt/openmpi - CFLAGS="-fno-omit-frame-pointer -g -O1 -fsanitize=address" + CFLAGS="-O2 -fno-omit-frame-pointer -g -fsanitize=address" LDFLAGS="-Wl,-rpath,/opt/openmpi/lib -fsanitize=address" working-directory: mpi-build @@ -131,6 +123,19 @@ jobs: env: CFLAGS: "-O0" + - name: Setting up ASAN environment + # LD_PRELOAD is needed to make sure ASAN is the first thing loaded + # as it will otherwise complain + # Leak detection is currently disabled because of the size of the report. + # The patcher is disabled because ASAN fails if code mmaps data at fixed + # memory addresses, see https://github.com/open-mpi/ompi/issues/12819 + # ODR violation detection is disabled until #13469 is fixed + run: | + echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV + echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV + echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV + - name: Test mpi4py (singleton) run: python test/main.py -v -x TestExcErrhandlerNull if: ${{ true }} @@ -161,28 +166,11 @@ jobs: if: ${{ true }} timeout-minutes: 10 - - name: Setting up ASAN environment - # LD_PRELOAD is needed to make sure ASAN is the first thing loaded - # as it will otherwise complain - # Leak detection is currently disabled because of the size of the report. - # The patcher is disabled because ASAN fails if code mmaps data at fixed - # memory addresses, see https://github.com/open-mpi/ompi/issues/12819 - # ODR violation detection is disabled until #13469 is fixed - run: | - echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV - echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV - echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV - echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV - - - name: Test mpi4py ASAN (np=1) - run: mpiexec -n 1 python test/main.py -v -x TestExcErrhandlerNull - if: ${{ true }} - timeout-minutes: 10 + - name: Show MPI (ASAN) + run: ompi_info - - name: Test mpi4py ASAN (np=4) - run: mpiexec -n 4 python test/main.py -v -f -x TestExcErrhandlerNull - if: ${{ true }} - timeout-minutes: 10 + - name: Show MPICC (ASAN) + run: mpicc -show - name: Disabling ASAN environment run: | From b8a4c15874f2664831cd50b1733f890b5f99a146 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 14 Jan 2026 18:16:09 -0500 Subject: [PATCH 4/4] ASAN: explicitly disable stack-use-after-return check This may reduce overhead, although according to https://github.com/google/sanitizers/wiki/addresssanitizerflags it should be disabled by default. Signed-off-by: Joseph Schuchart --- .github/workflows/ompi_mpi4py.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ompi_mpi4py.yaml b/.github/workflows/ompi_mpi4py.yaml index 0e5939118d2..bce5ed4f1df 100644 --- a/.github/workflows/ompi_mpi4py.yaml +++ b/.github/workflows/ompi_mpi4py.yaml @@ -125,14 +125,16 @@ jobs: - name: Setting up ASAN environment # LD_PRELOAD is needed to make sure ASAN is the first thing loaded - # as it will otherwise complain + # as it will otherwise complain. # Leak detection is currently disabled because of the size of the report. # The patcher is disabled because ASAN fails if code mmaps data at fixed - # memory addresses, see https://github.com/open-mpi/ompi/issues/12819 + # memory addresses, see https://github.com/open-mpi/ompi/issues/12819. # ODR violation detection is disabled until #13469 is fixed + # Disabling stack use after return detection to reduce slowdown, per + # https://github.com/llvm/llvm-project/issues/64190. run: | echo LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 >> $GITHUB_ENV - echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1 >> $GITHUB_ENV + echo ASAN_OPTIONS=detect_odr_violation=0,abort_on_error=1,detect_stack_use_after_return=0 >> $GITHUB_ENV echo LSAN_OPTIONS=detect_leaks=0,exitcode=0 >> $GITHUB_ENV echo OMPI_MCA_memory=^patcher >> $GITHUB_ENV