From 285a7375ecdcfb8ecd981ed22d3682768ca110ca Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 20 Mar 2026 11:27:27 +0000 Subject: [PATCH 1/5] fix Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da39fae989..f73c37f895c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -510,23 +510,51 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Build test archive + run: | + cargo nextest archive \ + --locked \ + -p vortex-cuda \ + --all-features \ + --target x86_64-unknown-linux-gnu \ + --archive-file nextest-archive.tar.zst - name: CUDA - memcheck env: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool memcheck --leak-check=full --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + run: | + cargo nextest run \ + --archive-file nextest-archive.tar.zst \ + --no-fail-fast \ + --verbose # TODO(joe): try to re-enable, This is hanging in CI. # - name: CUDA - racecheck # env: # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool racecheck --error-exitcode 1 - # run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + # run: | + # cargo nextest run \ + # --archive-file nextest-archive.tar.zst \ + # --no-fail-fast \ + # --verbose - name: CUDA - synccheck env: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool synccheck --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + run: | + cargo nextest run \ + --archive-file nextest-archive.tar.zst \ + --no-fail-fast \ + --verbose - name: CUDA - initcheck env: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool initcheck --error-exitcode 1 - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + run: | + cargo nextest run \ + --archive-file nextest-archive.tar.zst \ + --no-fail-fast \ + --verbose cuda-test-cudf: if: github.repository == 'vortex-data/vortex' From d5b55bc85ca718b4819a154cfe5b68f77dae986f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 20 Mar 2026 14:21:56 +0000 Subject: [PATCH 2/5] fix Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f73c37f895c..9808d0c8adc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -522,9 +522,25 @@ jobs: --all-features \ --target x86_64-unknown-linux-gnu \ --archive-file nextest-archive.tar.zst + - name: Create sanitizer runner wrapper + run: | + cat > /tmp/sanitizer-runner.sh << 'EOF' + #!/bin/bash + # nextest invokes the target runner for both listing and execution. + # compute-sanitizer fails during listing because no CUDA APIs are called. + # Detect the list phase and run the binary directly, bypassing the sanitizer. + for arg in "$@"; do + if [[ "$arg" == "--list" ]]; then + exec "$@" + fi + done + exec compute-sanitizer $SANITIZER_OPTS "$@" + EOF + chmod +x /tmp/sanitizer-runner.sh - name: CUDA - memcheck env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool memcheck --leak-check=full --error-exitcode 1 + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh + SANITIZER_OPTS: "--tool memcheck --leak-check=full --error-exitcode 1" run: | cargo nextest run \ --archive-file nextest-archive.tar.zst \ @@ -533,7 +549,8 @@ jobs: # TODO(joe): try to re-enable, This is hanging in CI. # - name: CUDA - racecheck # env: - # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool racecheck --error-exitcode 1 + # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh + # SANITIZER_OPTS: "--tool racecheck --error-exitcode 1" # run: | # cargo nextest run \ # --archive-file nextest-archive.tar.zst \ @@ -541,7 +558,8 @@ jobs: # --verbose - name: CUDA - synccheck env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool synccheck --error-exitcode 1 + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh + SANITIZER_OPTS: "--tool synccheck --error-exitcode 1" run: | cargo nextest run \ --archive-file nextest-archive.tar.zst \ @@ -549,7 +567,8 @@ jobs: --verbose - name: CUDA - initcheck env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: compute-sanitizer --tool initcheck --error-exitcode 1 + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh + SANITIZER_OPTS: "--tool initcheck --error-exitcode 1" run: | cargo nextest run \ --archive-file nextest-archive.tar.zst \ From fea2c2f36a75d51ebf940d3771d8a12528bd86ee Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 20 Mar 2026 14:33:32 +0000 Subject: [PATCH 3/5] fix Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9808d0c8adc..30cc5ace3d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,7 +534,14 @@ jobs: exec "$@" fi done - exec compute-sanitizer $SANITIZER_OPTS "$@" + compute-sanitizer $SANITIZER_OPTS "$@" + rc=$? + # Exit code 255 means "no instrumented API call" — the test didn't + # touch the GPU. This is harmless, not a sanitizer error. + if [[ $rc -eq 255 ]]; then + exit 0 + fi + exit $rc EOF chmod +x /tmp/sanitizer-runner.sh - name: CUDA - memcheck From 82a5c5a0fc96ad335fce28374fe10aa4a096de75 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 20 Mar 2026 15:00:59 +0000 Subject: [PATCH 4/5] ci: parallelize CUDA sanitizer checks with matrix strategy Run memcheck, synccheck, and initcheck as concurrent matrix jobs instead of sequentially. Each job uses sccache (100% cache hit rate) so the rebuild is fast, and total wall-clock time drops from ~5min to ~2.5min. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 4.6 Signed-off-by: Joe Isaacs --- .github/workflows/ci.yml | 90 +++++++++------------------------------- 1 file changed, 19 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30cc5ace3d4..9baa2ab1d82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -494,9 +494,22 @@ jobs: cuda-test-sanitizer: if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (sanitizer)" + name: "CUDA tests (${{ matrix.sanitizer }})" timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer + strategy: + fail-fast: false + matrix: + include: + - sanitizer: memcheck + runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" + # TODO(joe): try to re-enable racecheck, it is hanging in CI. + # - sanitizer: racecheck + # runner_flags: "--tool racecheck --error-exitcode 1" + - sanitizer: synccheck + runner_flags: "--tool synccheck --error-exitcode 1" + - sanitizer: initcheck + runner_flags: "--tool initcheck --error-exitcode 1" steps: - uses: runs-on/action@v2 with: @@ -510,77 +523,12 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest - - name: Build test archive - run: | - cargo nextest archive \ - --locked \ - -p vortex-cuda \ - --all-features \ - --target x86_64-unknown-linux-gnu \ - --archive-file nextest-archive.tar.zst - - name: Create sanitizer runner wrapper - run: | - cat > /tmp/sanitizer-runner.sh << 'EOF' - #!/bin/bash - # nextest invokes the target runner for both listing and execution. - # compute-sanitizer fails during listing because no CUDA APIs are called. - # Detect the list phase and run the binary directly, bypassing the sanitizer. - for arg in "$@"; do - if [[ "$arg" == "--list" ]]; then - exec "$@" - fi - done - compute-sanitizer $SANITIZER_OPTS "$@" - rc=$? - # Exit code 255 means "no instrumented API call" — the test didn't - # touch the GPU. This is harmless, not a sanitizer error. - if [[ $rc -eq 255 ]]; then - exit 0 - fi - exit $rc - EOF - chmod +x /tmp/sanitizer-runner.sh - - name: CUDA - memcheck + - name: Build tests + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run + - name: "CUDA - ${{ matrix.sanitizer }}" env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh - SANITIZER_OPTS: "--tool memcheck --leak-check=full --error-exitcode 1" - run: | - cargo nextest run \ - --archive-file nextest-archive.tar.zst \ - --no-fail-fast \ - --verbose - # TODO(joe): try to re-enable, This is hanging in CI. - # - name: CUDA - racecheck - # env: - # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh - # SANITIZER_OPTS: "--tool racecheck --error-exitcode 1" - # run: | - # cargo nextest run \ - # --archive-file nextest-archive.tar.zst \ - # --no-fail-fast \ - # --verbose - - name: CUDA - synccheck - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh - SANITIZER_OPTS: "--tool synccheck --error-exitcode 1" - run: | - cargo nextest run \ - --archive-file nextest-archive.tar.zst \ - --no-fail-fast \ - --verbose - - name: CUDA - initcheck - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: /tmp/sanitizer-runner.sh - SANITIZER_OPTS: "--tool initcheck --error-exitcode 1" - run: | - cargo nextest run \ - --archive-file nextest-archive.tar.zst \ - --no-fail-fast \ - --verbose + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu cuda-test-cudf: if: github.repository == 'vortex-data/vortex' From cd2fbee7eba70be5281e5e5f27d33f4f844fcf62 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 20 Mar 2026 15:39:42 +0000 Subject: [PATCH 5/5] chore[ci]: break up files Signed-off-by: Joe Isaacs --- .github/workflows/ci-cuda.yml | 154 +++++++++++++++++ .github/workflows/ci-java.yml | 67 ++++++++ .github/workflows/ci-python.yml | 130 +++++++++++++++ .github/workflows/ci.yml | 282 -------------------------------- 4 files changed, 351 insertions(+), 282 deletions(-) create mode 100644 .github/workflows/ci-cuda.yml create mode 100644 .github/workflows/ci-java.yml create mode 100644 .github/workflows/ci-python.yml diff --git a/.github/workflows/ci-cuda.yml b/.github/workflows/ci-cuda.yml new file mode 100644 index 00000000000..19254bbbe38 --- /dev/null +++ b/.github/workflows/ci-cuda.yml @@ -0,0 +1,154 @@ +name: CI / CUDA + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + cuda-build-lint: + if: github.repository == 'vortex-data/vortex' + name: "CUDA build & lint" + timeout-minutes: 40 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/check-rebuild + with: + command: >- + cargo build --locked --all-features --all-targets + -p vortex-cuda -p vortex-cub -p vortex-nvcomp + -p gpu-scan-cli -p vortex-test-e2e-cuda + - name: Clippy CUDA crates + run: | + cargo clippy --locked --all-features --all-targets \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p gpu-scan-cli \ + -p vortex-test-e2e-cuda \ + -- -D warnings + + cuda-test: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Rust Tests + env: + FLAT_LAYOUT_INLINE_ARRAY_NODE: true + run: | + cargo nextest run \ + --locked \ + -p vortex-file \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p vortex-test-e2e-cuda \ + --all-features \ + --no-fail-fast \ + --target x86_64-unknown-linux-gnu \ + --verbose + + cuda-test-sanitizer: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (${{ matrix.sanitizer }})" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer + strategy: + fail-fast: false + matrix: + include: + - sanitizer: memcheck + runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" + # TODO(joe): try to re-enable racecheck, it is hanging in CI. + # - sanitizer: racecheck + # runner_flags: "--tool racecheck --error-exitcode 1" + - sanitizer: synccheck + runner_flags: "--tool synccheck --error-exitcode 1" + - sanitizer: initcheck + runner_flags: "--tool initcheck --error-exitcode 1" + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build tests + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run + - name: "CUDA - ${{ matrix.sanitizer }}" + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" + run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + + cuda-test-cudf: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (cudf)" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build cudf test library + run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu + - name: Download and run cudf-test-harness + run: | + curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz + cd cudf-test-harness-x86_64 + compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so diff --git a/.github/workflows/ci-java.yml b/.github/workflows/ci-java.yml new file mode 100644 index 00000000000..c55ac630c61 --- /dev/null +++ b/.github/workflows/ci-java.yml @@ -0,0 +1,67 @@ +name: CI / Java + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + build-java: + name: "Java" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - run: ./gradlew test --parallel + working-directory: ./java + + check-java-publish-build: + runs-on: ${{ matrix.target.runs-on }} + container: + image: "ubuntu:20.04" + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + target: + - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } + - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: ./.github/actions/prepare-java-linux + - uses: actions/setup-java@v5 + with: + distribution: "corretto" + java-version: "17" + - uses: ./.github/actions/setup-rust + with: + targets: ${{ matrix.target.target }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - run: cargo build --package vortex-jni diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml new file mode 100644 index 00000000000..406f61c2e50 --- /dev/null +++ b/.github/workflows/ci-python.yml @@ -0,0 +1,130 @@ +name: CI / Python + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +env: + CARGO_TERM_COLOR: auto + RUST_BACKTRACE: 1 + +jobs: + python-lint: + name: "Python (lint)" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) + - name: Python Lint - Format + run: uvx ruff format --check . + - name: Python Lint - Ruff + run: uvx ruff check . + # PyRight needs the project for type information, so use uv run + - name: Python Lint - PyRight + env: + MATURIN_PEP517_ARGS: "--profile dev" + run: uv run basedpyright vortex-python + + python-test: + name: "Python (test)" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 40 + env: + RUST_LOG: "info,uv=debug" + MATURIN_PEP517_ARGS: "--profile dev" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + + - name: Pytest - Vortex + run: | + uv run --all-packages pytest --benchmark-disable -n auto test/ + working-directory: vortex-python/ + + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + + - name: Pytest Benchmarks - Vortex + run: | + bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ + working-directory: vortex-python/ + + - name: Doctest - PyVortex + run: | + uv run --all-packages make doctest + working-directory: docs/ + + - name: Ensure docs build - PyVortex + run: | + uv run --all-packages make html + working-directory: docs/ + + python-wheel-build: + name: "Python (wheel build)" + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref_name == 'develop' }} + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - uses: mlugg/setup-zig@v2.2.1 + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + with: + sync: false + prune-cache: false + - name: Ensure wheel and sdist can be built on Linux - PyVortex + shell: bash + run: | + echo "Clearing wheel target directory" + rm -rf ../target/wheels/ + + uv venv + uv tool run maturin@1.10 build --interpreter python3.11 --zig + uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist + + file_count=$(ls -1 ../target/wheels/ | wc -l) + + if [[ $file_count -ne 2 ]]; then + echo "Unexpected number of files detected ${file_count}:" + ls ../target/wheels/ + exit 1 + else + echo "Generated two files" + fi + working-directory: vortex-python/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9baa2ab1d82..ac8ab22598c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,113 +46,6 @@ jobs: -c .yamllint.yaml \ .github/ - python-lint: - name: "Python (lint)" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) - - name: Python Lint - Format - run: uvx ruff format --check . - - name: Python Lint - Ruff - run: uvx ruff check . - # PyRight needs the project for type information, so use uv run - - name: Python Lint - PyRight - env: - MATURIN_PEP517_ARGS: "--profile dev" - run: uv run basedpyright vortex-python - - python-test: - name: "Python (test)" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - env: - RUST_LOG: "info,uv=debug" - MATURIN_PEP517_ARGS: "--profile dev" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - - name: Pytest - Vortex - run: | - uv run --all-packages pytest --benchmark-disable -n auto test/ - working-directory: vortex-python/ - - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - - name: Pytest Benchmarks - Vortex - run: | - bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ - working-directory: vortex-python/ - - - name: Doctest - PyVortex - run: | - uv run --all-packages make doctest - working-directory: docs/ - - - name: Ensure docs build - PyVortex - run: | - uv run --all-packages make html - working-directory: docs/ - - python-wheel-build: - name: "Python (wheel build)" - runs-on: ubuntu-latest - timeout-minutes: 40 - steps: - - uses: actions/checkout@v6 - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref_name == 'develop' }} - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - uses: mlugg/setup-zig@v2.2.1 - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 - with: - sync: false - prune-cache: false - - name: Ensure wheel and sdist can be built on Linux - PyVortex - shell: bash - run: | - echo "Clearing wheel target directory" - rm -rf ../target/wheels/ - - uv venv - uv tool run maturin@1.10 build --interpreter python3.11 --zig - uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist - - file_count=$(ls -1 ../target/wheels/ | wc -l) - - if [[ $file_count -ne 2 ]]; then - echo "Unexpected number of files detected ${file_count}:" - ls ../target/wheels/ - exit 1 - else - echo "Generated two files" - fi - working-directory: vortex-python/ - rust-docs: name: "Rust (docs)" timeout-minutes: 40 @@ -425,137 +318,6 @@ jobs: --target x86_64-unknown-linux-gnu \ -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array - cuda-build-lint: - if: github.repository == 'vortex-data/vortex' - name: "CUDA build & lint" - timeout-minutes: 40 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/check-rebuild - with: - command: >- - cargo build --locked --all-features --all-targets - -p vortex-cuda -p vortex-cub -p vortex-nvcomp - -p gpu-scan-cli -p vortex-test-e2e-cuda - - name: Clippy CUDA crates - run: | - cargo clippy --locked --all-features --all-targets \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p gpu-scan-cli \ - -p vortex-test-e2e-cuda \ - -- -D warnings - - cuda-test: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest - - name: Rust Tests - env: - FLAT_LAYOUT_INLINE_ARRAY_NODE: true - run: | - cargo nextest run \ - --locked \ - -p vortex-file \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p vortex-test-e2e-cuda \ - --all-features \ - --no-fail-fast \ - --target x86_64-unknown-linux-gnu \ - --verbose - - cuda-test-sanitizer: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (${{ matrix.sanitizer }})" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer - strategy: - fail-fast: false - matrix: - include: - - sanitizer: memcheck - runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" - # TODO(joe): try to re-enable racecheck, it is hanging in CI. - # - sanitizer: racecheck - # runner_flags: "--tool racecheck --error-exitcode 1" - - sanitizer: synccheck - runner_flags: "--tool synccheck --error-exitcode 1" - - sanitizer: initcheck - runner_flags: "--tool initcheck --error-exitcode 1" - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build tests - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run - - name: "CUDA - ${{ matrix.sanitizer }}" - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" - run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - - cuda-test-cudf: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (cudf)" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build cudf test library - run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu - - name: Download and run cudf-test-harness - run: | - curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz - cd cudf-test-harness-x86_64 - compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so - rust-test-other: name: "Rust tests (${{ matrix.os }})" timeout-minutes: 40 @@ -612,23 +374,6 @@ jobs: alert-title: "Rust tests (${{ matrix.os }}) failed on develop" deduplication-key: ci-rust-test-${{ matrix.os }}-failure - build-java: - name: "Java" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 40 - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - run: ./gradlew test --parallel - working-directory: ./java - bench-codspeed: strategy: matrix: @@ -821,33 +566,6 @@ jobs: run: | find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} - check-java-publish-build: - runs-on: ${{ matrix.target.runs-on }} - container: - image: "ubuntu:20.04" - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - target: - - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } - - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/prepare-java-linux - - uses: actions/setup-java@v5 - with: - distribution: "corretto" - java-version: "17" - - uses: ./.github/actions/setup-rust - with: - targets: ${{ matrix.target.target }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - run: cargo build --package vortex-jni - compat-check: name: "Compat check" uses: ./.github/workflows/compat-validation.yml