diff --git a/.buildkite/ci_config.yaml b/.buildkite/ci_config.yaml
index b199e554a739..a60a4194e9b3 100644
--- a/.buildkite/ci_config.yaml
+++ b/.buildkite/ci_config.yaml
@@ -8,8 +8,8 @@ run_all_patterns:
- "CMakeLists.txt"
- "requirements/common.txt"
- "requirements/cuda.txt"
- - "requirements/build.txt"
- - "requirements/test.txt"
+ - "requirements/build/cuda.txt"
+ - "requirements/test/cuda.txt"
- "setup.py"
- "csrc/"
- "cmake/"
diff --git a/.buildkite/ci_config_intel.yaml b/.buildkite/ci_config_intel.yaml
index 375be84a396a..a1c0091e0f10 100644
--- a/.buildkite/ci_config_intel.yaml
+++ b/.buildkite/ci_config_intel.yaml
@@ -6,8 +6,8 @@ run_all_patterns:
- "CMakeLists.txt"
- "requirements/common.txt"
- "requirements/xpu.txt"
- - "requirements/build.txt"
- - "requirements/test.txt"
+ - "requirements/build/cuda.txt"
+ - "requirements/test/cuda.txt"
- "setup.py"
- "csrc/"
- "cmake/"
diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml
index e466e2a52444..9b1044443780 100644
--- a/.buildkite/hardware_tests/cpu.yaml
+++ b/.buildkite/hardware_tests/cpu.yaml
@@ -46,7 +46,7 @@ steps:
- tests/models/language/pooling/
commands:
- |
- bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
pytest -x -v -s tests/models/language/generation -m cpu_model
pytest -x -v -s tests/models/language/pooling -m cpu_model"
@@ -99,7 +99,7 @@ steps:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
- parallelism: 2
+ parallelism: 3
- label: "Arm CPU Test"
depends_on: []
diff --git a/.buildkite/image_build/image_build.sh b/.buildkite/image_build/image_build.sh
index 9131dfc71a0a..00ae34bba6d7 100755
--- a/.buildkite/image_build/image_build.sh
+++ b/.buildkite/image_build/image_build.sh
@@ -92,8 +92,8 @@ check_and_skip_if_image_exists() {
}
ecr_login() {
- aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
- aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
+ aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
+ aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
}
prepare_cache_tags() {
diff --git a/.buildkite/image_build/image_build_cpu.sh b/.buildkite/image_build/image_build_cpu.sh
index ccfe155fa2b7..035f070ab891 100755
--- a/.buildkite/image_build/image_build_cpu.sh
+++ b/.buildkite/image_build/image_build_cpu.sh
@@ -11,7 +11,7 @@ REPO=$2
BUILDKITE_COMMIT=$3
# authenticate with AWS ECR
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
diff --git a/.buildkite/image_build/image_build_cpu_arm64.sh b/.buildkite/image_build/image_build_cpu_arm64.sh
index ff3d11c8d599..b561e2c2e463 100755
--- a/.buildkite/image_build/image_build_cpu_arm64.sh
+++ b/.buildkite/image_build/image_build_cpu_arm64.sh
@@ -11,7 +11,7 @@ REPO=$2
BUILDKITE_COMMIT=$3
# authenticate with AWS ECR
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then
diff --git a/.buildkite/image_build/image_build_torch_nightly.sh b/.buildkite/image_build/image_build_torch_nightly.sh
new file mode 100755
index 000000000000..cbd08aa7bd0b
--- /dev/null
+++ b/.buildkite/image_build/image_build_torch_nightly.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build a vLLM test image with PyTorch nightly installed.
+# Called by the pipeline generator's "vLLM Against PyTorch Nightly" group.
+
+if [[ $# -lt 5 ]]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+REGISTRY=$1
+REPO=$2
+BUILDKITE_COMMIT=$3
+BRANCH=$4
+IMAGE_TAG=$5
+
+# --- Arguments ---
+echo "--- :mag: Arguments"
+echo "REGISTRY: ${REGISTRY}"
+echo "REPO: ${REPO}"
+echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
+echo "BRANCH: ${BRANCH}"
+echo "IMAGE_TAG: ${IMAGE_TAG}"
+
+# --- ECR login ---
+echo "--- :key: ECR login"
+aws ecr-public get-login-password --region us-east-1 \
+ | docker login --username AWS --password-stdin "$REGISTRY"
+aws ecr get-login-password --region us-east-1 \
+ | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
+
+# --- Set up buildx ---
+echo "--- :docker: Setting up buildx"
+docker buildx create --name vllm-builder --driver docker-container --use || true
+docker buildx inspect --bootstrap
+docker buildx ls
+
+# --- Skip if image already exists ---
+echo "--- :mag: Checking if image already exists"
+if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
+ echo "Image found: $IMAGE_TAG — skipping build"
+ exit 0
+fi
+echo "Image not found, proceeding with build..."
+
+# --- CUDA 13.0 for nightly builds ---
+# Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9
+NIGHTLY_CUDA_VERSION="13.0.2"
+NIGHTLY_BUILD_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-devel-ubuntu22.04"
+NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04"
+
+echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})"
+docker buildx build --file docker/Dockerfile \
+ --build-arg max_jobs=16 \
+ --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
+ --build-arg USE_SCCACHE=1 \
+ --build-arg PYTORCH_NIGHTLY=1 \
+ --build-arg CUDA_VERSION="${NIGHTLY_CUDA_VERSION}" \
+ --build-arg BUILD_BASE_IMAGE="${NIGHTLY_BUILD_BASE_IMAGE}" \
+ --build-arg FINAL_BASE_IMAGE="${NIGHTLY_FINAL_BASE_IMAGE}" \
+ --build-arg torch_cuda_arch_list="8.0 8.9 9.0 10.0 12.0" \
+ --tag "$IMAGE_TAG" \
+ --push \
+ --target test \
+ --progress plain .
+
+echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml
new file mode 100644
index 000000000000..c66576d40991
--- /dev/null
+++ b/.buildkite/intel_jobs/engine_intel.yaml
@@ -0,0 +1,21 @@
+group: Engine Intel
+depends_on:
+ - image-build-xpu
+steps:
+- label: Engine (1 GPU)
+ timeout_in_minutes: 30
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/v1/engine/
+ - tests/v1/engine/
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py'
diff --git a/.buildkite/intel_jobs/kernels_intel.yaml b/.buildkite/intel_jobs/kernels_intel.yaml
new file mode 100644
index 000000000000..66a8db25f02e
--- /dev/null
+++ b/.buildkite/intel_jobs/kernels_intel.yaml
@@ -0,0 +1,21 @@
+group: Kernels Intel
+depends_on:
+ - image-build-xpu
+steps:
+- label: vLLM IR Tests
+ timeout_in_minutes: 30
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/ir
+ - vllm/kernels
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s kernels/ir'
diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml
new file mode 100644
index 000000000000..729edd159cdf
--- /dev/null
+++ b/.buildkite/intel_jobs/lora_intel.yaml
@@ -0,0 +1,130 @@
+group: LoRA Intel
+depends_on:
+ - image-build-xpu
+steps:
+- label: LoRA Runtime + Utils
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s lora/test_layers.py &&
+ pytest -v -s lora/test_lora_checkpoints.py &&
+ (pytest -v -s lora/test_lora_functions.py --deselect="tests/lora/test_lora_functions.py::test_lora_functions_sync" --deselect="tests/lora/test_lora_functions.py::test_lora_functions_async" || true) &&
+ pytest -v -s lora/test_lora_huggingface.py &&
+ pytest -v -s lora/test_lora_manager.py &&
+ pytest -v -s lora/test_lora_utils.py &&
+ pytest -v -s lora/test_peft_helper.py &&
+ pytest -v -s lora/test_resolver.py &&
+ pytest -v -s lora/test_utils.py &&
+ (pytest -v -s lora/test_add_lora.py --deselect="tests/lora/test_add_lora.py::test_add_lora" || true) &&
+ (pytest -v -s lora/test_worker.py --deselect="tests/lora/test_worker.py::test_worker_apply_lora" || true)'
+
+- label: LoRA Fused/MoE Kernels
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s lora/test_fused_moe_lora_kernel.py &&
+ pytest -v -s lora/test_moe_lora_align_sum.py'
+
+- label: LoRA Punica Kernels
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ set -o pipefail &&
+ pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-2-2049-64-32-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-2-64000-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-1-102656-32-4-4]"'
+
+- label: LoRA Punica FP8/XPU Ops
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s lora/test_punica_ops_fp8.py &&
+ pytest -v -s lora/test_punica_xpu_ops.py'
+
+- label: LoRA Models
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
+ pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
+ pytest -v -s lora/test_qwen35_densemodel_lora.py &&
+ pytest -v -s lora/test_transformers_model.py'
+
+- label: LoRA Multimodal
+ timeout_in_minutes: 45
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/lora
+ - tests/lora
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'cd tests &&
+ pytest -v -s lora/test_default_mm_loras.py &&
+ (pytest -v -s lora/test_qwen3_unembed.py || true) &&
+ (pytest -v -s lora/test_qwenvl.py || true) &&
+ pytest -v -s lora/test_whisper.py'
diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml
new file mode 100644
index 000000000000..39634ccccacf
--- /dev/null
+++ b/.buildkite/intel_jobs/misc_intel.yaml
@@ -0,0 +1,55 @@
+group: Miscellaneous Intel
+depends_on:
+ - image-build-xpu
+steps:
+- label: V1 Core + KV + Metrics
+ timeout_in_minutes: 30
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/
+ - tests/v1/core
+ - tests/v1/executor
+ - tests/v1/kv_offload
+ - tests/v1/worker
+ - tests/v1/kv_connector/unit
+ - tests/v1/metrics
+ - tests/entrypoints/openai/correctness/test_lmeval.py
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'pip install -r requirements/kv_connectors.txt &&
+ export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+ cd tests &&
+ pytest -v -s v1/executor'
+
+- label: V1 Sample + Logits
+ timeout_in_minutes: 30
+ device: intel_gpu
+ no_plugin: true
+ working_dir: "."
+ env:
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
+ REPO: "vllm-ci-test-repo"
+ VLLM_TEST_DEVICE: "xpu"
+ source_file_dependencies:
+ - vllm/
+ - tests/v1/sample
+ - tests/v1/logits_processors
+ - tests/v1/test_oracle.py
+ - tests/v1/test_request.py
+ - tests/v1/test_outputs.py
+ commands:
+ - >-
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+ 'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+ cd tests &&
+ pytest -v -s v1/logits_processors &&
+ pytest -v -s v1/test_oracle.py &&
+ pytest -v -s v1/test_request.py &&
+ pytest -v -s v1/test_outputs.py'
diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml
index 295840440ad5..c59be699502f 100644
--- a/.buildkite/intel_jobs/test-intel.yaml
+++ b/.buildkite/intel_jobs/test-intel.yaml
@@ -35,6 +35,7 @@ steps:
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
@@ -61,4 +62,4 @@ steps:
pytest -v -s v1/structured_output &&
pytest -v -s v1/test_serial_utils.py &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py &&
- pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py'
+ pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py'
diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml
index 45b2996f7ead..8fce15680173 100644
--- a/.buildkite/release-pipeline.yaml
+++ b/.buildkite/release-pipeline.yaml
@@ -1,3 +1,13 @@
+# CUDA architecture lists — following PyTorch RELEASE.md
+# (https://github.com/pytorch/pytorch/blob/main/RELEASE.md)
+# SM86 included for broader Ampere coverage; SM89 for marlin fp8 support
+env:
+ CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
+ # aarch64 only architectures: 8.7 for Orin, 11.0 for Thor (since CUDA 13)
+ CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX"
+ CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
+ CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0"
+
steps:
- input: "Provide Release version here"
id: input-release-version
@@ -14,12 +24,10 @@ steps:
agents:
queue: arm64_cpu_queue_release
commands:
- # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
- # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- - "bash .buildkite/scripts/upload-nightly-wheels.sh"
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31"
env:
DOCKER_BUILDKIT: "1"
@@ -29,9 +37,7 @@ steps:
agents:
queue: arm64_cpu_queue_release
commands:
- # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
- # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
@@ -57,7 +63,7 @@ steps:
agents:
queue: cpu_queue_release
commands:
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31"
@@ -70,7 +76,7 @@ steps:
agents:
queue: cpu_queue_release
commands:
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
@@ -98,99 +104,105 @@ steps:
commands:
- "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
+ - block: "Unblock to build release Docker images"
+ depends_on: ~
+ key: block-build-release-images
+ if: build.env("NIGHTLY") != "1"
+
- group: "Build release Docker images"
key: "build-release-images"
+ depends_on: block-build-release-images
+ allow_dependency_failure: true
steps:
- - label: "Build release image - x86_64 - CUDA 12.9"
+ - label: "Build release image - x86_64 - CUDA 13.0"
depends_on: ~
id: build-release-image-x86
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
# re-tag to default image tag and push, just in case arm64 build fails
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
- - label: "Build release image - aarch64 - CUDA 12.9"
+ - label: "Build release image - aarch64 - CUDA 13.0"
depends_on: ~
id: build-release-image-arm64
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
- - label: "Build release image - x86_64 - CUDA 13.0"
+ - label: "Build release image - x86_64 - CUDA 12.9"
depends_on: ~
- id: build-release-image-x86-cuda-13-0
+ id: build-release-image-x86-cuda-12-9
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
# re-tag to default image tag and push, just in case arm64 build fails
- - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
- - label: "Build release image - aarch64 - CUDA 13.0"
+ - label: "Build release image - aarch64 - CUDA 12.9"
depends_on: ~
- id: build-release-image-arm64-cuda-13-0
+ id: build-release-image-arm64-cuda-12-9
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- # compute capability 12.0 for RTX-50 series / RTX PRO 6000 Blackwell, 12.1 for DGX Spark
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
- - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
+ - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
depends_on: ~
id: build-release-image-x86-ubuntu2404
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
- - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
+ - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
depends_on: ~
id: build-release-image-arm64-ubuntu2404
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
- - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
+ - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
depends_on: ~
- id: build-release-image-x86-cuda-13-0-ubuntu2404
+ id: build-release-image-x86-cuda-12-9-ubuntu2404
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
- - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
- - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
+ - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
depends_on: ~
- id: build-release-image-arm64-cuda-13-0-ubuntu2404
+ id: build-release-image-arm64-cuda-12-9-ubuntu2404
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
- block: "Build release image for x86_64 CPU"
key: block-cpu-release-image-build
@@ -231,7 +243,7 @@ steps:
- group: "Publish release images"
key: "publish-release-images"
steps:
- - label: "Create multi-arch manifest - CUDA 12.9"
+ - label: "Create multi-arch manifest - CUDA 13.0"
depends_on:
- build-release-image-x86
- build-release-image-arm64
@@ -243,7 +255,7 @@ steps:
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
- - label: "Annotate release workflow - CUDA 12.9"
+ - label: "Annotate release workflow - CUDA 13.0"
depends_on:
- create-multi-arch-manifest
id: annotate-release-workflow
@@ -252,19 +264,19 @@ steps:
commands:
- "bash .buildkite/scripts/annotate-release.sh"
- - label: "Create multi-arch manifest - CUDA 13.0"
+ - label: "Create multi-arch manifest - CUDA 12.9"
depends_on:
- - build-release-image-x86-cuda-13-0
- - build-release-image-arm64-cuda-13-0
- id: create-multi-arch-manifest-cuda-13-0
+ - build-release-image-x86-cuda-12-9
+ - build-release-image-arm64-cuda-12-9
+ id: create-multi-arch-manifest-cuda-12-9
agents:
queue: small_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend"
- - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend"
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
- - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
+ - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
depends_on:
- build-release-image-x86-ubuntu2404
- build-release-image-arm64-ubuntu2404
@@ -276,17 +288,17 @@ steps:
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend"
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
- - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
+ - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
depends_on:
- - build-release-image-x86-cuda-13-0-ubuntu2404
- - build-release-image-arm64-cuda-13-0-ubuntu2404
- id: create-multi-arch-manifest-cuda-13-0-ubuntu2404
+ - build-release-image-x86-cuda-12-9-ubuntu2404
+ - build-release-image-arm64-cuda-12-9-ubuntu2404
+ id: create-multi-arch-manifest-cuda-12-9-ubuntu2404
agents:
queue: small_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130-ubuntu2404 --amend"
- - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend"
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
- label: "Publish nightly multi-arch image to DockerHub"
depends_on:
@@ -306,16 +318,16 @@ steps:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- - label: "Publish nightly multi-arch image to DockerHub - CUDA 13.0"
+ - label: "Publish nightly multi-arch image to DockerHub - CUDA 12.9"
depends_on:
- - create-multi-arch-manifest-cuda-13-0
+ - create-multi-arch-manifest-cuda-12-9
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- - "bash .buildkite/scripts/push-nightly-builds.sh cu130"
+ - "bash .buildkite/scripts/push-nightly-builds.sh cu129"
# Clean up old nightly builds (keep only last 14)
- - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu130-nightly-"
+ - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu129-nightly-"
plugins:
- docker-login#v3.0.0:
username: vllmbot
@@ -617,6 +629,8 @@ steps:
- label: ":docker: Build release image - x86_64 - ROCm"
id: build-rocm-release-image
depends_on:
+ - step: block-build-release-images
+ allow_failure: true
- step: build-rocm-base-wheels
allow_failure: false
agents:
diff --git a/.buildkite/scripts/annotate-release.sh b/.buildkite/scripts/annotate-release.sh
index 2da9db2f2e5d..6f41d1cdda47 100755
--- a/.buildkite/scripts/annotate-release.sh
+++ b/.buildkite/scripts/annotate-release.sh
@@ -13,12 +13,12 @@ ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
To download the wheel (by commit):
\`\`\`
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl .
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_aarch64.whl .
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_x86_64.whl .
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_aarch64.whl .
-(Optional) For CUDA 13.0:
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_x86_64.whl .
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_aarch64.whl .
+(Optional) For CUDA 12.9:
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_x86_64.whl .
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_aarch64.whl .
(Optional) For CPU:
aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cpu-cp38-abi3-manylinux_2_35_x86_64.whl .
@@ -33,8 +33,8 @@ To download and upload the image:
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
docker pull public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
@@ -50,11 +50,11 @@ docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
docker push vllm/vllm-openai:latest-x86_64
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130 vllm/vllm-openai:x86_64-cu130
-docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:latest-x86_64-cu130
-docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
-docker push vllm/vllm-openai:latest-x86_64-cu130
-docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129 vllm/vllm-openai:x86_64-cu129
+docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129
+docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
+docker push vllm/vllm-openai:latest-x86_64-cu129
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64
docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64
@@ -62,11 +62,11 @@ docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
docker push vllm/vllm-openai:latest-aarch64
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130 vllm/vllm-openai:aarch64-cu130
-docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:latest-aarch64-cu130
-docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
-docker push vllm/vllm-openai:latest-aarch64-cu130
-docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129 vllm/vllm-openai:aarch64-cu129
+docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129
+docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
+docker push vllm/vllm-openai:latest-aarch64-cu129
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
## ROCm
@@ -104,11 +104,11 @@ docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${
docker manifest push vllm/vllm-openai:latest
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
-docker manifest rm vllm/vllm-openai:latest-cu130
-docker manifest create vllm/vllm-openai:latest-cu130 vllm/vllm-openai:latest-x86_64-cu130 vllm/vllm-openai:latest-aarch64-cu130
-docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
-docker manifest push vllm/vllm-openai:latest-cu130
-docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu130
+docker manifest rm vllm/vllm-openai:latest-cu129
+docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129
+docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
+docker manifest push vllm/vllm-openai:latest-cu129
+docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129
docker manifest rm vllm/vllm-openai-cpu:latest || true
docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
diff --git a/.buildkite/scripts/check-ray-compatibility.sh b/.buildkite/scripts/check-ray-compatibility.sh
index 1572fe94168d..b056d4403dbb 100644
--- a/.buildkite/scripts/check-ray-compatibility.sh
+++ b/.buildkite/scripts/check-ray-compatibility.sh
@@ -29,7 +29,7 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
TORCH_INDEX_URL=""
fi
else
- TORCH_INDEX_URL="https://download.pytorch.org/whl/cu129"
+ TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
fi
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/scripts/generate-and-upload-nightly-index.sh
index fa6eb979af53..88c4f5173139 100755
--- a/.buildkite/scripts/generate-and-upload-nightly-index.sh
+++ b/.buildkite/scripts/generate-and-upload-nightly-index.sh
@@ -9,7 +9,7 @@ set -ex
BUCKET="vllm-wheels"
INDICES_OUTPUT_DIR="indices"
-DEFAULT_VARIANT_ALIAS="cu129" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
+DEFAULT_VARIANT_ALIAS="cu130" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
PYTHON="${PYTHON_PROG:-python3}" # try to read from env var, otherwise use python3
SUBPATH=$BUILDKITE_COMMIT
S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
@@ -19,7 +19,7 @@ has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12)
if [[ "$has_new_python" -eq 0 ]]; then
# use new python from docker
docker pull python:3-slim
- PYTHON="docker run --rm -v $(pwd):/app -w /app python:3-slim python3"
+ PYTHON="docker run --rm -u $(id -u):$(id -g) -v $(pwd):/app -w /app python:3-slim python3"
fi
echo "Using python interpreter: $PYTHON"
diff --git a/.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
index 4ff15067aa0f..f12bb524d4cb 100644
--- a/.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
@@ -23,22 +23,22 @@ if [ "$failed_req" -ne 0 ]; then
exit 1
fi
-#echo "--- DP+TP"
-#vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 --max-model-len=4096 &
-#server_pid=$!
-#timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
-#vllm bench serve \
-# --backend vllm \
-# --dataset-name random \
-# --model meta-llama/Llama-3.2-3B-Instruct \
-# --num-prompts 20 \
-# --result-dir ./test_results \
-# --result-filename dp_pp.json \
-# --save-result \
-# --endpoint /v1/completions
-#kill -s SIGTERM $server_pid; wait $server_pid || true
-#failed_req=$(jq '.failed' ./test_results/dp_pp.json)
-#if [ "$failed_req" -ne 0 ]; then
-# echo "Some requests were failed!"
-# exit 1
-#fi
+echo "--- DP+TP"
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 --max-model-len=4096 &
+server_pid=$!
+timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
+vllm bench serve \
+ --backend vllm \
+ --dataset-name random \
+ --model meta-llama/Llama-3.2-3B-Instruct \
+ --num-prompts 20 \
+ --result-dir ./test_results \
+ --result-filename dp_pp.json \
+ --save-result \
+ --endpoint /v1/completions
+kill -s SIGTERM $server_pid; wait $server_pid || true
+failed_req=$(jq '.failed' ./test_results/dp_pp.json)
+if [ "$failed_req" -ne 0 ]; then
+ echo "Some requests were failed!"
+ exit 1
+fi
diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
index c2509a07b2c4..7166435ac1e9 100755
--- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
+++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
@@ -51,6 +51,7 @@ function cpu_tests() {
set -e
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
+ pytest -x -v -s tests/kernels/core/test_cpu_activation.py
pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic"
# basic online serving
diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test.sh b/.buildkite/scripts/hardware_ci/run-cpu-test.sh
index db75ad3083b2..27ec0068668f 100644
--- a/.buildkite/scripts/hardware_ci/run-cpu-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-cpu-test.sh
@@ -16,5 +16,5 @@ echo "--- :docker: Building Docker image"
docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
# Run the image, setting --shm-size=4g for tensor parallel.
-docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g "$IMAGE_NAME" \
+docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
diff --git a/.buildkite/scripts/hardware_ci/run-hpu-test.sh b/.buildkite/scripts/hardware_ci/run-hpu-test.sh
index 10df07b2000f..0b5d0af4b6f3 100644
--- a/.buildkite/scripts/hardware_ci/run-hpu-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-hpu-test.sh
@@ -42,7 +42,7 @@ WORKDIR /workspace/vllm
ENV no_proxy=localhost,127.0.0.1
ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
-RUN bash -c 'pip install -r <(sed "/^torch/d" requirements/build.txt)'
+RUN bash -c 'pip install -r <(sed "/^torch/d" requirements/build/cuda.txt)'
RUN VLLM_TARGET_DEVICE=empty pip install --no-build-isolation -e .
RUN pip install git+https://github.com/vllm-project/vllm-gaudi.git
diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh
index 836f0b849573..eae3b231b4be 100755
--- a/.buildkite/scripts/hardware_ci/run-intel-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh
@@ -25,22 +25,100 @@ export PYTHONPATH=".."
###############################################################################
cleanup_docker() {
+ # Share the same lock with image pull to avoid cleanup/pull races on one node.
+ local docker_lock="/tmp/docker-pull.lock"
+ exec 9>"$docker_lock"
+ flock 9
+
docker_root=$(docker info -f '{{.DockerRootDir}}')
if [ -z "$docker_root" ]; then
echo "Failed to determine Docker root directory." >&2
- exit 1
+ flock -u 9
+ return 1
fi
echo "Docker root directory: $docker_root"
disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
threshold=70
if [ "$disk_usage" -gt "$threshold" ]; then
- echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
- docker image prune -f
- docker volume prune -f && docker system prune --force --filter "until=72h" --all
- echo "Docker images and volumes cleanup completed."
+ echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..."
+ cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1
+ else
+ echo "Disk usage is below $threshold%. Checking old CI images anyway."
+ cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0
+ fi
+ echo "Old CI image cleanup completed."
+
+ flock -u 9
+}
+
+cleanup_old_ci_images() {
+ local repo_prefix="$1"
+ local current_image_ref="$2"
+ local ttl_hours="$3"
+ local aggressive_cleanup="$4"
+
+ if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then
+ echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'"
+ return 0
+ fi
+
+ if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then
+ echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72"
+ ttl_hours=72
+ fi
+
+ local now_epoch cutoff_epoch
+ now_epoch=$(date +%s)
+ cutoff_epoch=$((now_epoch - ttl_hours * 3600))
+
+ local -a used_image_ids
+ mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u)
+
+ local removed_count=0
+ local examined_count=0
+ declare -A seen_ids=()
+
+ while read -r image_ref image_id; do
+ [[ -z "$image_ref" || -z "$image_id" ]] && continue
+ ((examined_count++))
+
+ # Keep the image this job is going to use.
+ if [[ "$image_ref" == "$current_image_ref" ]]; then
+ continue
+ fi
+
+ # Avoid duplicate deletes when multiple tags point to same image id.
+ if [[ -n "${seen_ids[$image_id]:-}" ]]; then
+ continue
+ fi
+ seen_ids[$image_id]=1
+
+ # Never delete images that are used by any container on this node.
+ if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then
+ continue
+ fi
+
+ local created created_epoch
+ created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true)
+ [[ -z "$created" ]] && continue
+ created_epoch=$(date -d "$created" +%s 2>/dev/null || true)
+ [[ -z "$created_epoch" ]] && continue
+
+ if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then
+ if docker image rm -f "$image_id" >/dev/null 2>&1; then
+ ((removed_count++))
+ fi
+ fi
+ done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}')
+
+ # Also trim old dangling layers; this is safe and does not remove referenced images.
+ docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true
+
+ if [[ "$aggressive_cleanup" == "1" ]]; then
+ echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure."
else
- echo "Disk usage is below $threshold%. No cleanup needed."
+ echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)."
fi
}
@@ -240,7 +318,6 @@ fi
cleanup_docker
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
-aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
# --- Build or pull test image ---
IMAGE="${IMAGE_TAG_XPU:-${image_name}}"
@@ -266,8 +343,6 @@ fi
remove_docker_container() {
docker rm -f "${container_name}" || true
- docker image rm -f "${image_name}" || true
- docker system prune -f || true
}
trap remove_docker_container EXIT
@@ -283,6 +358,7 @@ docker run \
--ipc=host \
--privileged \
-v /dev/dri/by-path:/dev/dri/by-path \
+ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \
--entrypoint="" \
-e "HF_TOKEN=${HF_TOKEN:-}" \
-e "ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK:-}" \
diff --git a/.buildkite/scripts/hardware_ci/run-xpu-test.sh b/.buildkite/scripts/hardware_ci/run-xpu-test.sh
index 0467da7c2307..14bd08cfc1c4 100644
--- a/.buildkite/scripts/hardware_ci/run-xpu-test.sh
+++ b/.buildkite/scripts/hardware_ci/run-xpu-test.sh
@@ -12,9 +12,7 @@ docker build -t "${image_name}" -f docker/Dockerfile.xpu .
# Setup cleanup
remove_docker_container() {
- docker rm -f "${container_name}" || true;
- docker image rm -f "${image_name}" || true;
- docker system prune -f || true;
+ docker rm -f "${container_name}" || true
}
trap remove_docker_container EXIT
@@ -50,6 +48,6 @@ docker run \
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
pytest -v -s v1/structured_output
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py
- pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py -k "not (test_register_kv_caches and FLASH_ATTN and True)"
+ pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py
pytest -v -s v1/test_serial_utils.py
'
diff --git a/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh b/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh
new file mode 100755
index 000000000000..06743f16b687
--- /dev/null
+++ b/.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
+THRESHOLD=${1:-0.8}
+NUM_Q=${2:-1319}
+PORT=${3:-8050}
+OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
+mkdir -p "${OUT_DIR}"
+
+wait_for_server() {
+ local port=$1
+ timeout 600 bash -c '
+ until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
+ sleep 1
+ done'
+}
+
+MODEL="Qwen/Qwen3-30B-A3B-FP8"
+BACK="allgather_reducescatter"
+
+cleanup() {
+ if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+ kill "${SERVER_PID}" 2>/dev/null || true
+ for _ in {1..20}; do
+ kill -0 "${SERVER_PID}" 2>/dev/null || break
+ sleep 0.5
+ done
+ kill -9 "${SERVER_PID}" 2>/dev/null || true
+ fi
+}
+trap cleanup EXIT
+
+VLLM_DEEP_GEMM_WARMUP=skip \
+vllm serve "$MODEL" \
+--enforce-eager \
+--data-parallel-size 4 \
+--enable-expert-parallel \
+--enable-eplb \
+--all2all-backend "$BACK" \
+--eplb-config '{"window_size":20, "step_interval":100, "use_async":true}' \
+--trust-remote-code \
+--max-model-len 2048 \
+--port "$PORT" &
+SERVER_PID=$!
+wait_for_server "$PORT"
+
+TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
+OUT="${OUT_DIR}/${TAG}_${BACK}.json"
+python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
+python3 - <= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
+PY
diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml
index c7e9e13f2fd3..a8c9e4094387 100644
--- a/.buildkite/test-amd.yaml
+++ b/.buildkite/test-amd.yaml
@@ -1,8 +1,8 @@
# In this file, you can add more tests to run either by adding a new step or
# adding a new command to an existing step. See different options here for examples.
-# This script will be feed into Jinja template in `test-template-aws.j2` at
-# https://github.com/vllm-project/buildkite-ci/blob/main/scripts/test-template-aws.j2
+# This script will be feed into Jinja template in `test-template-amd.j2` at
+# https://github.com/vllm-project/buildkite-ci/blob/main/scripts/test-template-amd.j2
# to generate the final pipeline yaml file.
# Documentation
@@ -39,8 +39,8 @@
#####################################################################################################################################
# #
# IMPORTANT: #
-# * Currently AMD CI has MI250 agents, MI325 agents, and MI355 agents. All upcoming feature improvements are tracked in: #
-# https://github.com/vllm-project/vllm/issues/34994 #
+# * Currently AMD CI has MI250 agents, MI300 agents, MI325 agents, and MI355 agents. All upcoming feature improvements are #
+# tracked in: https://github.com/vllm-project/vllm/issues/34994 #
# #
#-----------------------------------------------------------------------------------------------------------------------------------#
# #
@@ -104,193 +104,195 @@
#####################################################################################################################################
-
-
steps:
+#########################################################################################################################################
+# #
+# MI250 (gfx90a) tests #
+# #
+#########################################################################################################################################
-#####################################################################################################################################
-# #
-# MI250 test definitions ( currently the test set is completely mirrored // TBD which tests are to be routed there ultimately) #
-# #
-#####################################################################################################################################
+#----------------------------------------------------- mi250 · basic_correctness -----------------------------------------------------#
-- label: Pytorch Nightly Dependency Override Check # TBD
+- label: Distributed Model Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- optional: true
- soft_fail: true
+ agent_pool: mi250_2
+ num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - requirements/nightly_torch_test.txt
+ - vllm/model_executor/model_loader/sharded_state_loader.py
+ - vllm/model_executor/models/
+ - vllm/model_executor/layers/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - tests/basic_correctness/
+ - tests/model_executor/model_loader/test_sharded_state_loader.py
+ - tests/models/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - bash standalone_tests/pytorch_nightly_dependency.sh
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
+ - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
+ - pytest models/language -v -s -m 'distributed(num_gpus=2)'
+ - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
+#-------------------------------------------------------- mi250 · benchmarks ---------------------------------------------------------#
-- label: Async Engine, Inputs, Utils, Worker # TBD
+- label: Benchmarks # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace/.buildkite"
source_file_dependencies:
- - vllm/
- - tests/detokenizer
- - tests/multimodal
- - tests/utils_
+ - benchmarks/
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s detokenizer
- - pytest -v -s -m 'not cpu_test' multimodal
- - pytest -v -s utils_
+ - bash scripts/run-benchmarks.sh
+#---------------------------------------------------------- mi250 · compile ----------------------------------------------------------#
-- label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD
+- label: PyTorch Compilation Unit Tests # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ torch_nightly: true
optional: true
- no_gpu: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/test_inputs.py
- - tests/test_outputs.py
- - tests/test_pooling_params.py
- - tests/test_ray_env.py
- - tests/multimodal
- - tests/renderers
- - tests/standalone_tests/lazy_imports.py
- - tests/tokenizers_
- - tests/tool_parsers
- - tests/transformers_utils
- - tests/config
+ - vllm/compilation/
+ - vllm/model_executor/layers/
+ - vllm/v1/worker/
+ - vllm/v1/attention/
+ - vllm/v1/cudagraph_dispatcher.py
+ - vllm/config/compilation.py
+ - csrc/
+ - tests/compile
+ - vllm/platforms/rocm.py
commands:
- - python3 standalone_tests/lazy_imports.py
- - pytest -v -s test_inputs.py
- - pytest -v -s test_outputs.py
- - pytest -v -s test_pooling_params.py
- - pytest -v -s test_ray_env.py
- - pytest -v -s -m 'cpu_test' multimodal
- - pytest -v -s renderers
- - pytest -v -s tokenizers_
- - pytest -v -s tool_parsers
- - pytest -v -s transformers_utils
- - pytest -v -s config
-
+ - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
-- label: Python-only Installation # TBD
+- label: PyTorch Fullgraph # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - tests/standalone_tests/python_only_compile.sh
- - setup.py
+ - vllm/compilation/
+ - vllm/model_executor/
+ - vllm/v1/attention/
+ - vllm/config/compilation.py
+ - csrc/
+ - tests/compile
- vllm/platforms/rocm.py
commands:
- - bash standalone_tests/python_only_compile.sh
-
+ - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
-- label: Basic Correctness # TBD
+- label: PyTorch Fullgraph Smoke Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- fast_check: true
+ optional: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/basic_correctness/test_basic_correctness
- - tests/basic_correctness/test_cpu_offload
- - tests/basic_correctness/test_cumem.py
- commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s basic_correctness/test_cumem.py
- - pytest -v -s basic_correctness/test_basic_correctness.py
- - pytest -v -s basic_correctness/test_cpu_offload.py
-
-
-- label: Entrypoints Unit Tests # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- fast_check: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/entrypoints
- - tests/entrypoints/
+ - vllm/compilation/
+ - vllm/model_executor/
+ - vllm/v1/attention/
+ - vllm/config/compilation.py
+ - csrc/
+ - tests/compile
- vllm/platforms/rocm.py
commands:
- - pytest -v -s entrypoints/openai/tool_parsers
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
-
+ - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\\\;"
-- label: Entrypoints Integration (LLM) # TBD
+- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- fast_check: true
- torch_nightly: true
+ agent_pool: mi250_2
+ num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/entrypoints/llm
- - tests/entrypoints/offline_mode
+ - vllm/compilation/
+ - vllm/distributed/
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/worker/worker_base.py
+ - vllm/v1/engine/
+ - vllm/v1/worker/
+ - tests/compile/fullgraph/test_basic_correctness.py
+ - tests/compile/test_wrapper.py
+ - tests/entrypoints/llm/test_collective_rpc.py
+ - vllm/platforms/rocm.py
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
- - pytest -v -s entrypoints/llm/test_generate.py
- - pytest -v -s entrypoints/offline_mode
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s entrypoints/llm/test_collective_rpc.py
+ - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
+ - pytest -v -s ./compile/test_wrapper.py
+#-------------------------------------------------------- mi250 · distributed --------------------------------------------------------#
-- label: Entrypoints Integration (API Server 2) # TBD
+- label: Distributed Comm Ops # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- fast_check: true
- torch_nightly: true
+ agent_pool: mi250_2
+ num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/entrypoints/rpc
- - tests/entrypoints/serve/instrumentator
- - tests/tool_use
+ - vllm/distributed
+ - tests/distributed
+ - vllm/platforms/rocm.py
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/serve/instrumentator
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
- - pytest -v -s tool_use
-
+ - pytest -v -s distributed/test_comm_ops.py
+ - pytest -v -s distributed/test_shm_broadcast.py
+ - pytest -v -s distributed/test_shm_buffer.py
+ - pytest -v -s distributed/test_shm_storage.py
-- label: Entrypoints Integration (Responses API) # TBD
+- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- fast_check: true
- torch_nightly: true
+ agent_pool: mi250_2
+ num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/entrypoints/openai/responses
+ - vllm/distributed/
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/worker/worker_base.py
+ - vllm/v1/engine/
+ - vllm/v1/worker/
+ - tests/distributed/
+ - tests/v1/shutdown
+ - tests/v1/worker/test_worker_memory_snapshot.py
+ - vllm/platforms/rocm.py
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai/responses
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+ - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
+ - pytest -v -s v1/worker/test_worker_memory_snapshot.py
-- label: EPLB Algorithm # TBD
+- label: Elastic EP Scaling Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ agent_pool: mi250_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/eplb
- - tests/distributed/test_eplb_algo.py
+ - vllm/distributed/
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/compilation/
+ - tests/distributed/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s distributed/test_eplb_algo.py
-
+ - pytest -v -s distributed/test_elastic_ep.py
- label: EPLB Execution # TBD
timeout_in_minutes: 180
@@ -307,8 +309,7 @@ steps:
- pytest -v -s distributed/test_eplb_execute.py
- pytest -v -s distributed/test_eplb_spec_decode.py
-
-- label: Elastic EP Scaling Test # TBD
+- label: Pipeline + Context Parallelism (4 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_4
@@ -318,1177 +319,1020 @@ steps:
- vllm/distributed/
- vllm/engine/
- vllm/executor/
- - vllm/compilation/
+ - vllm/model_executor/models/
+ - vllm/model_executor/layers/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
- tests/distributed/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s distributed/test_elastic_ep.py
+ - pytest -v -s distributed/test_pp_cudagraph.py
+ - pytest -v -s distributed/test_pipeline_parallel.py
+#---------------------------------------------------------- mi250 · engine -----------------------------------------------------------#
-- label: Metrics, Tracing (2 GPUs) # TBD
+- label: Engine # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/v1/tracing
+ - tests/engine
+ - tests/test_sequence
+ - tests/test_config
+ - tests/test_logger
+ - tests/test_vllm_port
commands:
- - "pip install \
- 'opentelemetry-sdk>=1.26.0' \
- 'opentelemetry-api>=1.26.0' \
- 'opentelemetry-exporter-otlp>=1.26.0' \
- 'opentelemetry-semantic-conventions-ai>=0.4.1'"
- - pytest -v -s v1/tracing
+ - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
+#----------------------------------------------------------- mi250 · evals -----------------------------------------------------------#
-- label: Regression # TBD
+- label: Multi-Modal Accuracy Eval (Small Models) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
+ optional: true
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- - vllm/
- - tests/test_regression
+ - vllm/multimodal/
+ - vllm/inputs/
+ - vllm/v1/core/
+ - vllm/platforms/rocm.py
+ - vllm/model_executor/model_loader/
commands:
- - pip install modelscope
- - pytest -v -s test_regression.py
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
+#--------------------------------------------------------- mi250 · examples ----------------------------------------------------------#
-- label: Engine # TBD
+- label: Examples # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace/examples"
source_file_dependencies:
- - vllm/
- - tests/engine
- - tests/test_sequence
- - tests/test_config
- - tests/test_logger
- - tests/test_vllm_port
+ - vllm/entrypoints
+ - vllm/multimodal
+ - examples/
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
+ - pip install tensorizer
+ # Basic
+ - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN
+ - python3 basic/offline_inference/generate.py --model facebook/opt-125m
+ - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
+ - python3 basic/offline_inference/classify.py
+ - python3 basic/offline_inference/embed.py
+ - python3 basic/offline_inference/score.py
+ # Multi-modal models
+ - python3 offline_inference/audio_language.py --seed 0
+ - python3 offline_inference/vision_language.py --seed 0
+ - python3 offline_inference/vision_language_multi_image.py --seed 0
+ - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
+ # Pooling models
+ - python3 pooling/embed/vision_embedding_offline.py --seed 0
+ # Features demo
+ - python3 offline_inference/prefix_caching.py
+ - python3 offline_inference/llm_engine_example.py
+ - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
+ - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
+ - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
+#---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------#
-- label: Engine (1 GPU) # TBD
+- label: Kernels Core Operation Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/
- - tests/v1/engine/
+ - csrc/
+ - tests/kernels/core
+ - tests/kernels/test_top_k_per_row.py
+ - tests/kernels/test_concat_mla_q.py
+ - vllm/model_executor/layers/rotary_embedding/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/engine/test_preprocess_error_handling.py
- - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
-
+ - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py
-- label: e2e Scheduling (1 GPU) # TBD
+- label: Kernels Helion Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/
- - tests/v1/e2e/general/
+ - vllm/utils/import_utils.py
+ - tests/kernels/helion/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/general/test_async_scheduling.py
-
+ - pip install helion==1.0.0
+ - pytest -v -s kernels/helion/
-- label: e2e Core (1 GPU) # TBD
+- label: Kernels Mamba Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/
- - tests/v1/e2e/general/
+ - csrc/mamba/
+ - tests/kernels/mamba
+ - vllm/model_executor/layers/mamba/ops
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
+ - pytest -v -s kernels/mamba
+#----------------------------------------------------------- mi250 · lora ------------------------------------------------------------#
-- label: Spec Decode Speculators + MTP # TBD
+- label: LoRA %N # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ parallelism: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
- - vllm/model_executor/layers/
- - vllm/transformers_utils/configs/speculators/
- - tests/v1/e2e/spec_decode/
+ - vllm/lora
+ - tests/lora
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
+ - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
+#------------------------------------------------------ mi250 · model_executor -------------------------------------------------------#
-- label: Spec Decode Ngram + Suffix # TBD
+- label: Model Executor # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
- - vllm/model_executor/layers/
- - tests/v1/e2e/spec_decode/
+ - vllm/engine/arg_utils.py
+ - vllm/config/model.py
+ - vllm/model_executor
+ - tests/model_executor
+ - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
+ - apt-get update && apt-get install -y curl libsodium23
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -v -s model_executor -m '(not slow_test)'
+ - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
+#---------------------------------------------------------- mi250 · models -----------------------------------------------------------#
-- label: Spec Decode Draft Model # TBD
+- label: Basic Models Test (Other CPU) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ no_gpu: true
optional: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
+ - vllm/
+ - tests/models/test_utils.py
+ - tests/models/test_vision.py
+ commands:
+ - pytest -v -s models/test_utils.py models/test_vision.py
+
+- label: Basic Models Tests (Extra Initialization) %N # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
+ agent_pool: mi250_1
+ torch_nightly: true
+ parallelism: 2
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/model_executor/models/
- vllm/model_executor/layers/
- - tests/v1/e2e/spec_decode/
+ - tests/models/test_initialization.py
+ - tests/models/registry.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
-
+ - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
-- label: V1 e2e (2 GPUs) # TBD
+- label: Basic Models Tests (Initialization) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- optional: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/v1/e2e
+ - tests/models/test_initialization.py
+ - tests/models/registry.py
commands:
- - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-
+ - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
-- label: V1 Sample + Logits # TBD
- timeout_in_minutes: 60
+- label: Basic Models Tests (Other) # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- optional: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/v1/sample
- - tests/v1/logits_processors
- - tests/v1/test_oracle.py
- - tests/v1/test_request.py
- - tests/v1/test_outputs.py
+ - tests/models/test_terratorch.py
+ - tests/models/test_transformers.py
+ - tests/models/test_registry.py
commands:
- - pytest -v -s v1/sample
- - pytest -v -s v1/logits_processors
- - pytest -v -s v1/test_oracle.py
- - pytest -v -s v1/test_request.py
- - pytest -v -s v1/test_outputs.py
-
+ - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
-- label: V1 Core + KV + Metrics # TBD
- timeout_in_minutes: 60
+- label: Language Models Test (MTEB) # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/v1/core
- - tests/v1/executor
- - tests/v1/kv_offload
- - tests/v1/worker
- - tests/v1/kv_connector/unit
- - tests/v1/metrics
- - tests/entrypoints/openai/correctness/test_lmeval.py
+ - tests/models/language/pooling_mteb_test
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - pytest -v -s -m 'not cpu_test' v1/core
- - pytest -v -s v1/executor
- - pytest -v -s v1/kv_offload
- - pytest -v -s v1/worker
- - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
- - pytest -v -s -m 'not cpu_test' v1/metrics
- - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
- - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-
+ - pytest -v -s models/language/pooling_mteb_test
-- label: V1 Speculative Decoding (slow) # TBD
+- label: Language Models Test (PPL) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/model_executor/models/
- - vllm/v1/attention/
- - vllm/model_executor/layers/
- - tests/v1/spec_decode/
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/models/language/generation_ppl_test
commands:
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_eagle.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_extract_hidden_states.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_max_len.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_mtp.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_ngram.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_speculators_eagle3.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_tree_attention.py
-
+ - pytest -v -s models/language/generation_ppl_test
-- label: V1 attention (H100-MI250) # TBD
+- label: Language Models Tests (Extra Standard) %N # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ torch_nightly: true
+ parallelism: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/config/attention.py
- - vllm/model_executor/layers/attention
- - vllm/v1/attention
- - tests/v1/attention
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/model_executor/layers/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - tests/models/language/pooling/test_embedding.py
+ - tests/models/language/generation/test_common.py
+ - tests/models/language/pooling/test_classification.py
- vllm/_aiter_ops.py
- - vllm/envs.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/attention
-
+ - pip freeze | grep -E 'torch'
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
-- label: V1 others (CPU) # TBD
+- label: Multi-Modal Models (Extended Generation 2) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- no_gpu: true
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/v1
+ - tests/models/multimodal/generation
commands:
- - pytest -v -s -m 'cpu_test' v1/core
- - pytest -v -s v1/structured_output
- - pytest -v -s v1/test_serial_utils.py
- - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
- - pytest -v -s -m 'cpu_test' v1/metrics
-
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
-- label: Examples # TBD
+- label: Multi-Modal Models (Extended Pooling) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- optional: true
- working_dir: "/vllm-workspace/examples"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/entrypoints
- - vllm/multimodal
- - examples/
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/models/multimodal/pooling
commands:
- - pip install tensorizer
- # Basic
- - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN
- - python3 basic/offline_inference/generate.py --model facebook/opt-125m
- - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
- - python3 basic/offline_inference/classify.py
- - python3 basic/offline_inference/embed.py
- - python3 basic/offline_inference/score.py
- # Multi-modal models
- - python3 offline_inference/audio_language.py --seed 0
- - python3 offline_inference/vision_language.py --seed 0
- - python3 offline_inference/vision_language_multi_image.py --seed 0
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
- # Pooling models
- - python3 pooling/embed/vision_embedding_offline.py --seed 0
- # Features demo
- - python3 offline_inference/prefix_caching.py
- - python3 offline_inference/llm_engine_example.py
- - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
- - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
- - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
-
+ - pytest -v -s models/multimodal/pooling -m 'not core_model'
-- label: Platform Tests (CUDA) # TBD
+- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/cuda
+ - tests/models/multimodal
commands:
- - pytest -v -s cuda/test_cuda_context.py
- - pytest -v -s cuda/test_platform_no_cuda_init.py
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
+ - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
+#---------------------------------------------------------- mi250 · plugins ----------------------------------------------------------#
-- label: Samplers Test # TBD
+- label: Plugin Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ agent_pool: mi250_2
+ num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/layers
- - vllm/sampling_metadata.py
- - vllm/v1/sample/
- - vllm/beam_search.py
- - tests/samplers
- - tests/conftest.py
- - vllm/_aiter_ops.py
+ - vllm/plugins/
+ - tests/plugins/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s samplers
+ # BEGIN: platform plugin and general plugin tests, all the code in-between runs on dummy platform
+ - pip install -e ./plugins/vllm_add_dummy_platform
+ - pytest -v -s plugins_tests/test_platform_plugins.py
+ - pip uninstall vllm_add_dummy_platform -y
+ # END: platform plugin tests
+ # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin
+ - pip install -e ./plugins/prithvi_io_processor_plugin
+ - pytest -v -s plugins_tests/test_io_processor_plugins.py
+ - pytest -v -s plugins_tests/test_terratorch_io_processor_plugins.py
+ - pip uninstall prithvi_io_processor_plugin -y
+ # END: `io_processor` plugins test
+ # BEGIN: `bge_m3_sparse io_processor` test
+ - pip install -e ./plugins/bge_m3_sparse_plugin
+ - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
+ - pip uninstall bge_m3_sparse_plugin -y
+ # END: `bge_m3_sparse io_processor` test
+ # BEGIN: `stat_logger` plugins test
+ - pip install -e ./plugins/vllm_add_dummy_stat_logger
+ - pytest -v -s plugins_tests/test_stats_logger_plugins.py
+ - pip uninstall dummy_stat_logger -y
+ # END: `stat_logger` plugins test
+ # BEGIN: other tests
+ - pytest -v -s plugins_tests/test_scheduler_plugins.py
+ - pip install -e ./plugins/vllm_add_dummy_model
+ - pytest -v -s distributed/test_distributed_oot.py
+ - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py
+ - pytest -v -s models/test_oot_registration.py
+ - pytest -v -s plugins/lora_resolvers
+#------------------------------------------------------------ mi250 · v1 -------------------------------------------------------------#
-- label: LoRA %N # TBD
+- label: Batch Invariance (H100-MI250) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- parallelism: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/lora
- - tests/lora
+ - vllm/v1/attention
+ - vllm/model_executor/layers
+ - tests/v1/determinism/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py
-
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pip install pytest-timeout pytest-forked
+ - pytest -v -s v1/determinism/test_batch_invariance.py
+ - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-- label: PyTorch Compilation Unit Tests # TBD
+- label: Cudagraph # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- torch_nightly: true
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/compilation/
- - vllm/model_executor/layers/
- - vllm/v1/worker/
- - vllm/v1/attention/
+ - tests/v1/cudagraph
- vllm/v1/cudagraph_dispatcher.py
- vllm/config/compilation.py
- - csrc/
- - tests/compile
+ - vllm/compilation
- vllm/platforms/rocm.py
commands:
- - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
-
+ - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
+ - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
-- label: PyTorch Fullgraph Smoke Test # TBD
+- label: e2e Core (1 GPU) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
- torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/compilation/
- - vllm/model_executor/
- - vllm/v1/attention/
- - vllm/config/compilation.py
- - csrc/
- - tests/compile
+ - vllm/v1/
+ - tests/v1/e2e/general/
- vllm/platforms/rocm.py
commands:
- - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\\\;"
-
+ - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
-- label: PyTorch Fullgraph # TBD
+- label: e2e Scheduling (1 GPU) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
- torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/compilation/
- - vllm/model_executor/
- - vllm/v1/attention/
- - vllm/config/compilation.py
- - csrc/
- - tests/compile
+ - vllm/v1/
+ - tests/v1/e2e/general/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
-
+ - pytest -v -s v1/e2e/general/test_async_scheduling.py
-- label: Cudagraph # TBD
+- label: Engine (1 GPU) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - tests/v1/cudagraph
- - vllm/v1/cudagraph_dispatcher.py
- - vllm/config/compilation.py
- - vllm/compilation
+ - vllm/v1/
+ - tests/v1/engine/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
- - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
-
+ - pytest -v -s v1/engine/test_preprocess_error_handling.py
+ - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
-- label: Kernels Core Operation Test # TBD
+- label: Spec Decode Draft Model # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - tests/kernels/core
- - tests/kernels/test_top_k_per_row.py
- - tests/kernels/test_concat_mla_q.py
- - vllm/model_executor/layers/rotary_embedding/
- - vllm/_aiter_ops.py
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s kernels/core kernels/test_top_k_per_row.py
-
+ - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
-- label: Kernels Mamba Test # TBD
+- label: Spec Decode Speculators + MTP # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/mamba/
- - tests/kernels/mamba
- - vllm/model_executor/layers/mamba/ops
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s kernels/mamba
-
+ - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
-- label: Kernels Helion Test # TBD
+- label: V1 attention (H100-MI250) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/utils/import_utils.py
- - tests/kernels/helion/
+ - vllm/config/attention.py
+ - vllm/model_executor/layers/attention
+ - vllm/v1/attention
+ - tests/v1/attention
+ - vllm/_aiter_ops.py
+ - vllm/envs.py
- vllm/platforms/rocm.py
commands:
- - pip install helion==0.3.3
- - pytest -v -s kernels/helion/
-
+ - pytest -v -s v1/attention
-- label: Model Executor # TBD
- timeout_in_minutes: 180
+- label: V1 Core + KV + Metrics # TBD
+ timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
- torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/engine/arg_utils.py
- - vllm/config/model.py
- - vllm/model_executor
- - tests/model_executor
- - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - apt-get update && apt-get install -y curl libsodium23
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s model_executor -m '(not slow_test)'
- - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
-
-
-- label: Benchmarks # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/vllm-workspace/.buildkite"
- source_file_dependencies:
- - benchmarks/
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/core
+ - tests/v1/executor
+ - tests/v1/kv_offload
+ - tests/v1/worker
+ - tests/v1/kv_connector/unit
+ - tests/v1/metrics
+ - tests/entrypoints/openai/correctness/test_lmeval.py
commands:
- - bash scripts/run-benchmarks.sh
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - pytest -v -s -m 'not cpu_test' v1/core
+ - pytest -v -s v1/executor
+ - pytest -v -s v1/kv_offload
+ - pytest -v -s v1/worker
+ - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
+ - pytest -v -s -m 'not cpu_test' v1/metrics
+ - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
+ - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-- label: Benchmarks CLI Test # TBD
- timeout_in_minutes: 180
+- label: V1 Sample + Logits # TBD
+ timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/benchmarks/
+ - tests/v1/sample
+ - tests/v1/logits_processors
+ - tests/v1/test_oracle.py
+ - tests/v1/test_request.py
+ - tests/v1/test_outputs.py
commands:
- - pytest -v -s benchmarks/
-
+ - pytest -v -s v1/sample
+ - pytest -v -s v1/logits_processors
+ - pytest -v -s v1/test_oracle.py
+ - pytest -v -s v1/test_request.py
+ - pytest -v -s v1/test_outputs.py
-- label: OpenAI API correctness # TBD
+- label: Distributed DP Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
+ agent_pool: mi250_2
+ num_gpus: 2
+ optional: true
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/entrypoints/openai/
- - vllm/model_executor/models/whisper.py
- - vllm/model_executor/layers/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/worker/worker_base.py
+ - vllm/v1/engine/
+ - vllm/v1/worker/
+ - tests/v1/distributed
+ - tests/entrypoints/openai/test_multi_api_servers.py
- vllm/platforms/rocm.py
- - vllm/model_executor/model_loader/
commands:
- - bash ../tools/install_torchcodec_rocm.sh || exit 1
- - pytest -s entrypoints/openai/correctness/
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
+ - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
+- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
+ agent_pool: mi250_2
+ num_gpus: 2
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - vllm/v1/worker/kv_connector_model_runner_mixin.py
+ - tests/v1/kv_connector/nixl_integration/
+ - vllm/platforms/rocm.py
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
-- label: Basic Models Tests (Initialization) # TBD
+- label: V1 e2e (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
+ agent_pool: mi250_2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/test_initialization.py
- - tests/models/registry.py
+ - tests/v1/e2e
commands:
- - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
-
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-- label: Basic Models Tests (Extra Initialization) %N # TBD
+- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
- parallelism: 2
+ agent_pool: mi250_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/layers/
- - tests/models/test_initialization.py
- - tests/models/registry.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+#------------------------------------------------------------- mi250 · misc ------------------------------------------------------------#
-- label: Basic Models Tests (Other) # TBD
+- label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
- torch_nightly: true
+ optional: true
+ no_gpu: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/test_terratorch.py
- - tests/models/test_transformers.py
- - tests/models/test_registry.py
+ - tests/test_inputs.py
+ - tests/test_outputs.py
+ - tests/test_pooling_params.py
+ - tests/test_ray_env.py
+ - tests/multimodal
+ - tests/renderers
+ - tests/standalone_tests/lazy_imports.py
+ - tests/tokenizers_
+ - tests/tool_parsers
+ - tests/transformers_utils
+ - tests/config
commands:
- - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
+ - python3 standalone_tests/lazy_imports.py
+ - pytest -v -s test_inputs.py
+ - pytest -v -s test_outputs.py
+ - pytest -v -s test_pooling_params.py
+ - pytest -v -s test_ray_env.py
+ - pytest -v -s -m 'cpu_test' multimodal
+ - pytest -v -s renderers
+ - pytest -v -s tokenizers_
+ - pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py --ignore=reasoning/test_gemma4_reasoning_parser.py
+ - pytest -v -s tool_parsers
+ - pytest -v -s transformers_utils
+ - pytest -v -s config
+#########################################################################################################################################
+# #
+# MI300 (gfx942) tests #
+# #
+#########################################################################################################################################
-- label: Basic Models Test (Other CPU) # TBD
+#----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------#
+
+- label: Basic Correctness # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- no_gpu: true
- optional: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/test_utils.py
- - tests/models/test_vision.py
+ - tests/basic_correctness/test_basic_correctness
+ - tests/basic_correctness/test_cpu_offload
+ - tests/basic_correctness/test_cumem.py
commands:
- - pytest -v -s models/test_utils.py models/test_vision.py
-
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -v -s basic_correctness/test_cumem.py
+ - pytest -v -s basic_correctness/test_basic_correctness.py
+ - pytest -v -s basic_correctness/test_cpu_offload.py
-- label: Language Models Tests (Extra Standard) %N # TBD
+- label: Distributed Model Tests (2 GPUs) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
- parallelism: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
+ - vllm/model_executor/model_loader/sharded_state_loader.py
- vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- vllm/model_executor/layers/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- - tests/models/language/pooling/test_embedding.py
- - tests/models/language/generation/test_common.py
- - tests/models/language/pooling/test_classification.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+ - tests/basic_correctness/
+ - tests/model_executor/model_loader/test_sharded_state_loader.py
+ - tests/models/
commands:
- - pip freeze | grep -E 'torch'
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
+ - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
+ - pytest models/language -v -s -m 'distributed(num_gpus=2)'
+ - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
+ - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
+#-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------#
-- label: Language Models Test (PPL) # TBD
+- label: Benchmarks # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/.buildkite"
+ source_file_dependencies:
+ - benchmarks/
+ - vllm/platforms/rocm.py
+ commands:
+ - bash scripts/run-benchmarks.sh
+
+- label: Benchmarks CLI Test # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/language/generation_ppl_test
+ - tests/benchmarks/
commands:
- - pytest -v -s models/language/generation_ppl_test
+ - pytest -v -s benchmarks/
+#---------------------------------------------------------- mi300 · compile ----------------------------------------------------------#
-- label: Language Models Test (Extended Pooling) # TBD
+- label: Fusion E2E Config Sweep (H100-MI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ num_gpus: 1
+ working_dir: "/vllm-workspace/"
source_file_dependencies:
- - vllm/
- - tests/models/language/pooling
+ - csrc/quantization/
+ - vllm/compilation/
+ - vllm/model_executor/layers/layernorm.py
+ - vllm/model_executor/layers/activation.py
+ - vllm/model_executor/layers/attention/attention.py
+ - vllm/model_executor/layers/quantization/input_quant_fp8.py
+ - tests/compile/fusions_e2e/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s models/language/pooling -m 'not core_model'
+ - rocm-smi
+ - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
+- label: Fusion E2E Quick (H100-MI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ num_gpus: 1
+ working_dir: "/vllm-workspace/"
+ source_file_dependencies:
+ - csrc/quantization/
+ - vllm/model_executor/
+ - vllm/v1/attention/
+ - vllm/compilation/
+ - tests/compile/fusions_e2e/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ commands:
+ - rocm-smi
+ # Run all models and attn backends but only Inductor partition and native custom ops
+ - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
+ # Different from CUDA, Qwen requires +rms_norm and +quant_fp8 as rms+quant fusion is only supported on AITER
+ - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
-- label: Language Models Test (MTEB) # TBD
+- label: PyTorch Compilation Passes Unit Tests # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/language/pooling_mteb_test
+ - tests/compile/passes
commands:
- - pytest -v -s models/language/pooling_mteb_test
-
+ - pytest -s -v compile/passes --ignore compile/passes/distributed
-- label: Multi-Modal Processor (CPU) # TBD
+- label: Pytorch Nightly Dependency Override Check # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- no_gpu: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
+ soft_fail: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal
- - tests/models/registry.py
+ - requirements/test/nightly-torch.txt
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
-
+ - bash standalone_tests/pytorch_nightly_dependency.sh
-- label: Multi-Modal Accuracy Eval (Small Models) # TBD
+- label: Distributed Compile Unit Tests (2xH100-2xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- optional: true
- working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
+ working_dir: "/vllm-workspace/"
source_file_dependencies:
- - vllm/multimodal/
- - vllm/inputs/
- - vllm/v1/core/
+ - vllm/compilation/
+ - vllm/model_executor/layers
+ - tests/compile/passes/distributed/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- - vllm/model_executor/model_loader/
commands:
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
+ - export VLLM_TEST_CLEAN_GPU_MEMORY=1
+ - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
+ - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
+#----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
-- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
+- label: Platform Tests (CUDA) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/multimodal
+ - tests/cuda
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
+ - pytest -v -s cuda/test_cuda_context.py
+ - pytest -v -s cuda/test_platform_no_cuda_init.py
+#-------------------------------------------------------- mi300 · detokenizer --------------------------------------------------------#
-- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
+- label: Async Engine, Inputs, Utils, Worker # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
-
-
-- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
-
-
-- label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- torch_nightly: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/multimodal/generation
- - tests/models/multimodal/test_mapping.py
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
- - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
-
-
-- label: Multi-Modal Models (Extended Generation 1) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
- - tests/models/multimodal/test_mapping.py
+ - tests/detokenizer
+ - tests/multimodal
+ - tests/utils_
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- - pytest -v -s models/multimodal/test_mapping.py
+ - pytest -v -s detokenizer
+ - pytest -v -s -m 'not cpu_test' multimodal
+ - pytest -v -s utils_
+#-------------------------------------------------------- mi300 · distributed --------------------------------------------------------#
-- label: Multi-Modal Models (Extended Generation 2) # TBD
+- label: EPLB Algorithm # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
+ - vllm/distributed/eplb
+ - tests/distributed/test_eplb_algo.py
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
-
+ - pytest -v -s distributed/test_eplb_algo.py
+ - pytest -v -s distributed/test_eplb_utils.py
-- label: Multi-Modal Models (Extended Generation 3) # TBD
+- label: Distributed Tests (2xH100-2xMI250) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/vllm-workspace/tests"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
+ working_dir: "/vllm-workspace/"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
+ - vllm/distributed/
+ - vllm/v1/distributed/
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - tests/distributed/test_context_parallel.py
+ - examples/offline_inference/data_parallel.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s tests/distributed/test_context_parallel.py
+ - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
-- label: Multi-Modal Models (Extended Pooling) # TBD
+- label: Distributed Tests (4xA100-4xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/multimodal/pooling
commands:
- - pytest -v -s models/multimodal/pooling -m 'not core_model'
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s distributed/test_custom_all_reduce.py
+ - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
+ - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
+ - pytest -v -s -x lora/test_mixtral.py
-- label: Distributed Comm Ops # TBD
+- label: Distributed Torchrun + Examples (4 GPUs) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- num_gpus: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed
- - tests/distributed
+ - vllm/distributed/
+ - tests/distributed/test_torchrun_example.py
+ - tests/distributed/test_torchrun_example_moe.py
+ - examples/rl/
+ - tests/examples/offline_inference/data_parallel.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s distributed/test_comm_ops.py
- - pytest -v -s distributed/test_shm_broadcast.py
- - pytest -v -s distributed/test_shm_buffer.py
- - pytest -v -s distributed/test_shm_storage.py
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
+ - PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
+ - TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
+ - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
+ - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
+ - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
+ - python3 ../examples/offline_inference/data_parallel.py --enforce-eager
+ # rlhf examples
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_nccl.py
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_ipc.py
-- label: Distributed DP Tests (2 GPUs) # TBD
+- label: Elastic EP Scaling Test # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/
- vllm/engine/
- vllm/executor/
- - vllm/worker/worker_base.py
- - vllm/v1/engine/
- - vllm/v1/worker/
- - tests/v1/distributed
- - tests/entrypoints/openai/test_multi_api_servers.py
+ - vllm/compilation/
+ - tests/distributed/
- vllm/platforms/rocm.py
commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
- - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
- - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
-
+ - pytest -v -s distributed/test_elastic_ep.py
-- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
+- label: RayExecutorV2 (4 GPUs) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
- optional: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/compilation/
- - vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/worker/worker_base.py
- - vllm/v1/engine/
- - vllm/v1/worker/
- - tests/compile/fullgraph/test_basic_correctness.py
- - tests/compile/test_wrapper.py
- - tests/entrypoints/llm/test_collective_rpc.py
+ - vllm/v1/executor/ray_executor_v2.py
+ - vllm/v1/executor/abstract.py
+ - vllm/v1/executor/multiproc_executor.py
+ - tests/distributed/test_ray_v2_executor.py
+ - tests/distributed/test_ray_v2_executor_e2e.py
+ - tests/distributed/test_pipeline_parallel.py
+ - tests/basic_correctness/test_basic_correctness.py
- vllm/platforms/rocm.py
commands:
+ - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
- export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s entrypoints/llm/test_collective_rpc.py
- - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
- - pytest -v -s ./compile/test_wrapper.py
+ - pytest -v -s distributed/test_ray_v2_executor.py
+ - pytest -v -s distributed/test_ray_v2_executor_e2e.py
+ - pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
+ - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
-
-- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD
+- label: Distributed Tests (8xH100-8xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_8
+ num_gpus: 8
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
+ - examples/offline_inference/torchrun_dp_example.py
+ - vllm/config/parallel.py
- vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/worker/worker_base.py
- - vllm/v1/engine/
- - vllm/v1/worker/
- - tests/distributed/
- - tests/v1/shutdown
- - tests/v1/worker/test_worker_memory_snapshot.py
+ - vllm/v1/engine/llm_engine.py
+ - vllm/v1/executor/uniproc_executor.py
+ - vllm/v1/worker/gpu_worker.py
- vllm/platforms/rocm.py
commands:
- export TORCH_NCCL_BLOCKING_WAIT=1
- - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
- - pytest -v -s v1/worker/test_worker_memory_snapshot.py
+ - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
+#-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
-- label: Distributed Model Tests (2 GPUs) # TBD
+- label: Entrypoints Integration (API Server 2) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/model_loader/sharded_state_loader.py
- - vllm/model_executor/models/
- - vllm/model_executor/layers/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - tests/basic_correctness/
- - tests/model_executor/model_loader/test_sharded_state_loader.py
- - tests/models/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/entrypoints/rpc
+ - tests/entrypoints/serve/instrumentator
+ - tests/tool_use
commands:
- - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
- - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
- - pytest models/language -v -s -m 'distributed(num_gpus=2)'
- - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
- - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -v -s entrypoints/serve/instrumentator
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
+ - pytest -v -s tool_use
+- label: Entrypoints Integration (API Server openai - Part 1) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
+ torch_nightly: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/entrypoints/openai
+ - tests/entrypoints/test_chat_utils
+ commands:
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
-- label: Plugin Tests (2 GPUs) # TBD
+- label: Entrypoints Integration (API Server openai - Part 2) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
- optional: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/plugins/
- - tests/plugins/
- - vllm/platforms/rocm.py
- commands:
- # BEGIN: platform plugin and general plugin tests, all the code in-between runs on dummy platform
- - pip install -e ./plugins/vllm_add_dummy_platform
- - pytest -v -s plugins_tests/test_platform_plugins.py
- - pip uninstall vllm_add_dummy_platform -y
- # END: platform plugin tests
- # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin
- - pip install -e ./plugins/prithvi_io_processor_plugin
- - pytest -v -s plugins_tests/test_io_processor_plugins.py
- - pip uninstall prithvi_io_processor_plugin -y
- # END: `io_processor` plugins test
- # BEGIN: `bge_m3_sparse io_processor` test
- - pip install -e ./plugins/bge_m3_sparse_plugin
- - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
- - pip uninstall bge_m3_sparse_plugin -y
- # END: `bge_m3_sparse io_processor` test
- # BEGIN: `stat_logger` plugins test
- - pip install -e ./plugins/vllm_add_dummy_stat_logger
- - pytest -v -s plugins_tests/test_stats_logger_plugins.py
- - pip uninstall dummy_stat_logger -y
- # END: `stat_logger` plugins test
- # BEGIN: other tests
- - pytest -v -s plugins_tests/test_scheduler_plugins.py
- - pip install -e ./plugins/vllm_add_dummy_model
- - pytest -v -s distributed/test_distributed_oot.py
- - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py
- - pytest -v -s models/test_oot_registration.py
- - pytest -v -s plugins/lora_resolvers
- # END: other tests
-
-
-- label: Pipeline + Context Parallelism (4 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/model_executor/models/
- - vllm/model_executor/layers/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - tests/distributed/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s distributed/test_pp_cudagraph.py
- - pytest -v -s distributed/test_pipeline_parallel.py
-
-
-- label: Ray Dependency Compatibility Check # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_1
- working_dir: "/"
- source_file_dependencies:
- - requirements/
- - setup.py
- - vllm/platforms/rocm.py
- commands:
- - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
-
-
-- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-
-- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-
-- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_2
- num_gpus: 2
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - vllm/v1/worker/kv_connector_model_runner_mixin.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
-
-
-- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - CROSS_LAYERS_BLOCKS=True ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-
-- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi250_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - HYBRID_SSM=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-
-- label: Distributed Tests (2 GPUs)(H100-MI250) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
- agent_pool: mi325_2
- num_gpus: 2
- working_dir: "/vllm-workspace/"
- source_file_dependencies:
- - vllm/distributed/
- - vllm/v1/distributed/
- - vllm/model_executor/layers/fused_moe/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - tests/distributed/test_context_parallel.py
- - examples/offline_inference/data_parallel.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s tests/distributed/test_context_parallel.py
- - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
-
-
-#####################################################################################################################################
-# #
-# gfx942 #
-# #
-#####################################################################################################################################
-
-
-- label: Entrypoints Integration (LLM) # 13.1m
- timeout_in_minutes: 22
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- fast_check: true
- torch_nightly: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/entrypoints/llm
- - tests/entrypoints/offline_mode
- commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
- - pytest -v -s entrypoints/llm/test_generate.py
- - pytest -v -s entrypoints/offline_mode
-
-
-- label: Entrypoints Integration (API Server openai - Part 1) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- fast_check: true
- torch_nightly: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/entrypoints/openai
- - tests/entrypoints/test_chat_utils
- commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
-
-
-- label: Entrypoints Integration (API Server openai - Part 2) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- fast_check: true
- torch_nightly: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/entrypoints/openai
- - tests/entrypoints/test_chat_utils
+ - vllm/
+ - tests/entrypoints/openai
+ - tests/entrypoints/test_chat_utils
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- pytest -v -s entrypoints/openai/speech_to_text/
- pytest -v -s entrypoints/test_chat_utils.py
-
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
@@ -1498,33 +1342,30 @@ steps:
- tests/entrypoints/test_chat_utils
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
-
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
-- label: Entrypoints Integration (API Server 2) #26.9m
- timeout_in_minutes: 45
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Entrypoints Integration (LLM) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/entrypoints/rpc
- - tests/entrypoints/serve/instrumentator
- - tests/tool_use
+ - tests/entrypoints/llm
+ - tests/entrypoints/offline_mode
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/serve/instrumentator
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
- - pytest -v -s tool_use
-
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
+ - pytest -v -s entrypoints/llm/test_generate.py
+ - pytest -v -s entrypoints/offline_mode
-- label: Entrypoints Integration (Pooling) # 22.8m
- timeout_in_minutes: 48
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Entrypoints Integration (Pooling) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
@@ -1535,446 +1376,260 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/pooling
-
-- label: Distributed Torchrun + Examples (4 GPUs) # TBD
- timeout_in_minutes: 80
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/
- - tests/distributed/test_torchrun_example.py
- - tests/distributed/test_torchrun_example_moe.py
- - examples/rl/
- - tests/examples/offline_inference/data_parallel.py
- - vllm/platforms/rocm.py
- commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
- - PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
- - TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
- - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
- - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
- - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
- - python3 ../examples/offline_inference/data_parallel.py --enforce-eager
- # rlhf examples
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_nccl.py
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_ipc.py
-
-
-- label: Distributed DP Tests (4 GPUs) # TBD
- timeout_in_minutes: 60
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/
- - tests/v1/distributed
- - tests/v1/engine/test_engine_core_client.py
- - tests/distributed/test_utils
- - vllm/platforms/rocm.py
- commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
- - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
- - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
- - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
- - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
- - pytest -v -s distributed/test_utils.py
-
-
-- label: Distributed Compile + Comm (4 GPUs) # TBD
- timeout_in_minutes: 40
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/
- - tests/distributed/test_pynccl
- - tests/distributed/test_events
- - tests/compile/fullgraph/test_basic_correctness.py
- - tests/distributed/test_symm_mem_allreduce.py
- - tests/distributed/test_multiproc_executor.py
- - vllm/platforms/rocm.py
- commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s compile/fullgraph/test_basic_correctness.py
- - pytest -v -s distributed/test_pynccl.py
- - pytest -v -s distributed/test_events.py
- - pytest -v -s distributed/test_symm_mem_allreduce.py
- - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
-
-
-- label: Distributed Tests (8 GPUs)(H100-MI325) # 6.4m
- timeout_in_minutes: 10
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_8
- num_gpus: 8
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - examples/offline_inference/torchrun_dp_example.py
- - vllm/config/parallel.py
- - vllm/distributed/
- - vllm/v1/engine/llm_engine.py
- - vllm/v1/executor/uniproc_executor.py
- - vllm/v1/worker/gpu_worker.py
- - vllm/platforms/rocm.py
- commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
-
-
-- label: Elastic EP Scaling Test # TBD
+- label: Entrypoints Integration (Responses API) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/compilation/
- - tests/distributed/
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s distributed/test_elastic_ep.py
-
-
-- label: Engine # 11.3m
- timeout_in_minutes: 35
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
+ torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/engine
- - tests/test_sequence
- - tests/test_config
- - tests/test_logger
- - tests/test_vllm_port
- commands:
- - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
-
-
-- label: Engine (1 GPU) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/v1/engine/
- - tests/v1/engine/
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s v1/engine/test_preprocess_error_handling.py
- - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
-
-
-- label: e2e Scheduling (1 GPU) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/v1/
- - tests/v1/e2e/general/
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s v1/e2e/general/test_async_scheduling.py
-
-
-- label: e2e Core (1 GPU) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/v1/
- - tests/v1/e2e/
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
-
-
-- label: Spec Decode Eagle # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
- - vllm/model_executor/layers/
- - tests/v1/e2e/spec_decode/
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
-
-
-- label: Spec Decode Speculators + MTP # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
- - vllm/model_executor/layers/
- - vllm/transformers_utils/configs/speculators/
- - tests/v1/e2e/spec_decode/
- - vllm/platforms/rocm.py
+ - tests/entrypoints/openai/responses
commands:
- - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
-
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -v -s entrypoints/openai/responses
-- label: Spec Decode Ngram + Suffix # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
+- label: Entrypoints Unit Tests # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ fast_check: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
- - vllm/model_executor/layers/
- - tests/v1/e2e/spec_decode/
+ - vllm/entrypoints
+ - tests/entrypoints/
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
-
+ - pytest -v -s entrypoints/openai/tool_parsers
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
-- label: Spec Decode Draft Model # TBD
+- label: OpenAI API correctness # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/v1/worker/gpu/spec_decode/
- - vllm/model_executor/model_loader/
- - vllm/v1/sample/
+ - csrc/
+ - vllm/entrypoints/openai/
+ - vllm/model_executor/models/whisper.py
- vllm/model_executor/layers/
- - tests/v1/e2e/spec_decode/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+ - vllm/model_executor/model_loader/
commands:
- - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
+ - bash ../tools/install_torchcodec_rocm.sh || exit 1
+ - pytest -s entrypoints/openai/correctness/
+#----------------------------------------------------------- mi300 · evals -----------------------------------------------------------#
-- label: V1 e2e (2 GPUs) # 7.1m
- timeout_in_minutes: 12
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
- optional: true
- working_dir: "/vllm-workspace/tests"
+- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ num_gpus: 1
+ working_dir: "/vllm-workspace"
source_file_dependencies:
- - vllm/
- - tests/v1/e2e
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/model_executor/layers/quantization/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/backends/mla/
+ - vllm/v1/attention/selector.py
+ - .buildkite/scripts/scheduled_integration_test/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-
+ - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
-- label: V1 e2e (4 GPUs) # 52.6m
- timeout_in_minutes: 106
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- optional: true
+- label: LM Eval Small Models # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1/e2e
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
-
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
-- label: V1 e2e (4xH100-4xMI325) # TBD
+- label: LM Eval Small Models (MI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- optional: true
- source_file_dependencies:
- - vllm/v1/attention/backends/utils.py
- - vllm/v1/worker/gpu_model_runner.py
- - tests/v1/e2e/test_hybrid_chunked_prefill.py
- commands:
- - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py
-
-
-- label: V1 Spec Decode # TBD
- timeout_in_minutes: 40
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/tests"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- - vllm/
- - tests/v1/spec_decode
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s -m 'not slow_test' v1/spec_decode
-
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
-- label: V1 Sample + Logits # TBD
- timeout_in_minutes: 40
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: GPQA Eval (GPT-OSS) (2xH100-2xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1/sample
- - tests/v1/logits_processors
- - tests/v1/test_oracle.py
- - tests/v1/test_request.py
- - tests/v1/test_outputs.py
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ - tests/evals/gpt_oss/
commands:
- - pytest -v -s v1/sample
- - pytest -v -s v1/logits_processors
- - pytest -v -s v1/test_oracle.py
- - pytest -v -s v1/test_request.py
- - pytest -v -s v1/test_outputs.py
-
+ - uv pip install --system 'gpt-oss[eval]==0.0.5'
+ - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
-- label: V1 Core + KV + Metrics # TBD
- timeout_in_minutes: 40
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
+- label: LM Eval Small Models (2xB200-2xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1/core
- - tests/v1/executor
- - tests/v1/kv_offload
- - tests/v1/worker
- - tests/v1/kv_connector/unit
- - tests/v1/metrics
- - tests/entrypoints/openai/correctness/test_lmeval.py
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - pytest -v -s -m 'not cpu_test' v1/core
- - pytest -v -s v1/executor
- - pytest -v -s v1/kv_offload
- - pytest -v -s v1/worker
- - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
- - pytest -v -s -m 'not cpu_test' v1/metrics
- - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
- # - export HSA_NO_SCRATCH_RECLAIM=1
- - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
-- label: V1 Speculative Decoding (slow) # TBD
+- label: DeepSeek V2-Lite Accuracy (4xH100-4xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
optional: true
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace"
source_file_dependencies:
- - vllm/v1/spec_decode/
- vllm/model_executor/models/
- - vllm/v1/attention/
- - vllm/model_executor/layers/
- - tests/v1/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/distributed/eplb
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/model_executor/layers/quantization/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/backends/mla/
+ - vllm/v1/attention/selector.py
+ - .buildkite/scripts/scheduled_integration_test/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_eagle.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_extract_hidden_states.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_max_len.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_mtp.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_ngram.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_speculators_eagle3.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_tree_attention.py
-
+ - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
-- label: Acceptance Length Test (Large Models) # TBD
+- label: LM Eval Large Models (4xA100-4xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
optional: true
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- - vllm/v1/spec_decode/
- - vllm/model_executor/models/mlp_speculator.py
- - tests/v1/spec_decode/test_acceptance_length.py
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
- - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
-
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
-- label: V1 attention (H100-MI325) # 14.5m
- timeout_in_minutes: 40
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/tests"
+- label: Qwen3-30B-A3B-FP8-block Accuracy (4xH100-4xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ optional: true
+ working_dir: "/vllm-workspace"
source_file_dependencies:
- - vllm/config/attention.py
- - vllm/model_executor/layers/attention
- - vllm/v1/attention
- - tests/v1/attention
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/model_executor/layers/quantization/
+ - vllm/distributed/eplb
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- - vllm/envs.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/attention
-
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
-- label: Batch Invariance (H100-MI325) # 5.2m
- timeout_in_minutes: 12
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
optional: true
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace"
source_file_dependencies:
- - vllm/v1/attention
- - vllm/model_executor/layers
- - tests/v1/determinism/
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/spec_decode/
+ - vllm/distributed/eplb
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/model_executor/layers/quantization/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - .buildkite/scripts/scheduled_integration_test/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pip install pytest-timeout pytest-forked
- - pytest -v -s v1/determinism/test_batch_invariance.py
- - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-
+ - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
-- label: V1 others (CPU) # 10.4m
- timeout_in_minutes: 28
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- no_gpu: true
+- label: LM Eval Large Models (8xH200-8xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_8
optional: true
+ num_gpus: 8
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/model_executor/layers/quantization/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/model_executor/layers/layernorm.py
+ - csrc/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ - tests/evals/
commands:
- - pytest -v -s -m 'cpu_test' v1/core
- - pytest -v -s v1/structured_output
- - pytest -v -s v1/test_serial_utils.py
- - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
- - pytest -v -s -m 'cpu_test' v1/metrics
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
+#--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
-- label: Examples # 24.5m
- timeout_in_minutes: 55
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Examples # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/examples"
source_file_dependencies:
@@ -2005,54 +1660,12 @@ steps:
- python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
- python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
+#---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------#
-- label: Platform Tests (CUDA) # 5.0m
- timeout_in_minutes: 9
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/cuda
- commands:
- - pytest -v -s cuda/test_cuda_context.py
- - pytest -v -s cuda/test_platform_no_cuda_init.py
-
-
-- label: PyTorch Compilation Passes Unit Tests # TBD
+- label: Kernels Attention Test %N # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/compile/passes
- commands:
- - pytest -s -v compile/passes --ignore compile/passes/distributed
-
-
-- label: Kernels Core Operation Test # 26.8m
- timeout_in_minutes: 38
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - csrc/
- - tests/kernels/core
- - tests/kernels/test_top_k_per_row.py
- - tests/kernels/test_concat_mla_q.py
- - vllm/model_executor/layers/rotary_embedding/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - pytest -v -s kernels/core kernels/test_top_k_per_row.py
-
-
-- label: Kernels Attention Test %N # 17.7m
- timeout_in_minutes: 28
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
parallelism: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
@@ -2066,29 +1679,26 @@ steps:
commands:
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-
-- label: Kernels Quantization Test %N # 15.2m
- timeout_in_minutes: 24
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- parallelism: 2
+- label: Kernels Core Operation Test # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/quantization/
- - vllm/model_executor/layers/quantization
- - tests/kernels/quantization
- - tests/kernels/quantization/test_rocm_skinny_gemms.py
+ - csrc/
+ - tests/kernels/core
+ - tests/kernels/test_top_k_per_row.py
+ - tests/kernels/test_concat_mla_q.py
+ - vllm/model_executor/layers/rotary_embedding/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- - vllm/model_executor/kernels/
commands:
- - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-
+ - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py
- label: Kernels MoE Test %N # TBD
- timeout_in_minutes: 19
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
parallelism: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
@@ -2105,11 +1715,27 @@ steps:
- pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+- label: Kernels Quantization Test %N # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ parallelism: 2
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - csrc/quantization/
+ - vllm/model_executor/layers/quantization
+ - tests/kernels/quantization
+ - tests/kernels/quantization/test_rocm_skinny_gemms.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ - vllm/model_executor/kernels/
+ commands:
+ - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-- label: Kernels FP8 MoE Test # TBD
+- label: Kernels FP8 MoE Test (2xH100-2xMI300) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
@@ -2123,40 +1749,217 @@ steps:
commands:
- pytest -v -s kernels/moe/test_deepep_moe.py
+#----------------------------------------------------------- mi300 · lora ------------------------------------------------------------#
-- label: ROCm AITER Ops Test # TBD
+- label: LoRA TP (Distributed) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/_aiter_ops.py
- - vllm/envs.py
+ - vllm/lora
+ - tests/lora
- vllm/platforms/rocm.py
- - tests/rocm/aiter/
- - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
- - vllm/v1/attention/selector.py
commands:
- - pytest -v -s rocm/aiter/
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+ - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+ - pytest -v -s -x lora/test_chatglm3_tp.py
+ - pytest -v -s -x lora/test_llama_tp.py
+ - pytest -v -s -x lora/test_llm_with_multi_loras.py
+ - pytest -v -s -x lora/test_olmoe_tp.py
+ - pytest -v -s -x lora/test_gptoss_tp.py
+ - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
+#---------------------------------------------------------- mi300 · models -----------------------------------------------------------#
-- label: Benchmarks # 8.2m
- timeout_in_minutes: 20
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Language Models Test (Extended Pooling) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/language/pooling
+ commands:
+ - pytest -v -s models/language/pooling -m 'not core_model'
+
+- label: Language Models Tests (Standard) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
- working_dir: "/vllm-workspace/.buildkite"
+ torch_nightly: true
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - benchmarks/
+ - vllm/
+ - tests/models/language
+ commands:
+ - pip freeze | grep -E 'torch'
+ - pytest -v -s models/language -m 'core_model and (not slow_test)'
+
+- label: Multi-Modal Models (Extended Generation 1) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal/generation
+ - tests/models/multimodal/test_mapping.py
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+ - pytest -v -s models/multimodal/test_mapping.py
+
+- label: Multi-Modal Models (Extended Generation 2) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal/generation
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
+
+- label: Multi-Modal Models (Extended Generation 3) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal/generation
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
+
+- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ torch_nightly: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
+ - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
+
+- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ torch_nightly: true
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal/generation
+ - tests/models/multimodal/test_mapping.py
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
+ - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
+
+- label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ torch_nightly: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal/generation
+ - tests/models/multimodal/test_mapping.py
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
+ - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
+ - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
+
+- label: Multi-Modal Processor # 1h 42m
+ timeout_in_minutes: 138
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal
+ - tests/models/registry.py
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/processing/test_tensor_schema.py
+
+- label: Multi-Modal Processor (CPU) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ no_gpu: true
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal
+ - tests/models/registry.py
+ commands:
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
+
+- label: Quantized Models Test # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/model_executor/layers/quantization
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ - tests/models/quantization
+ - vllm/model_executor/model_loader/
+ commands:
+ - pytest -v -s models/quantization
+
+- label: Transformers Nightly Models # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/"
+ source_file_dependencies:
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/multimodal/
+ - vllm/model_executor/layers/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+ - tests/models/
+ - examples/
commands:
- - bash scripts/run-benchmarks.sh
+ - pip install --upgrade git+https://github.com/huggingface/transformers
+ - pytest -v -s tests/models/test_initialization.py
+ - pytest -v -s tests/models/test_transformers.py
+ - pytest -v -s tests/models/multimodal/processing/
+ - pytest -v -s tests/models/multimodal/test_mapping.py
+ - python3 examples/basic/offline_inference/chat.py
+ - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
+#------------------------------------------------------- mi300 · quantization --------------------------------------------------------#
-- label: Quantization # 36.1m
- timeout_in_minutes: 60
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Quantization # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- csrc/
@@ -2177,261 +1980,293 @@ steps:
- uv pip install --system conch-triton-kernels
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
+#----------------------------------------------------------- mi300 · rocm ------------------------------------------------------------#
-- label: Language Models Tests (Standard) # 22.8m
- timeout_in_minutes: 38
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- torch_nightly: true
+- label: ROCm AITER Ops Test # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/language
+ - vllm/_aiter_ops.py
+ - vllm/envs.py
+ - vllm/platforms/rocm.py
+ - tests/rocm/aiter/
+ - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
+ - vllm/v1/attention/selector.py
commands:
- - pip freeze | grep -E 'torch'
- - pytest -v -s models/language -m 'core_model and (not slow_test)'
+ - pytest -v -s rocm/aiter/
+#--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------#
-- label: Language Models Tests (Hybrid) %N # 34.9m
- timeout_in_minutes: 55
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- torch_nightly: true
- parallelism: 2
+- label: Samplers Test # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/language/generation
+ - vllm/model_executor/layers
+ - vllm/sampling_metadata.py
+ - vllm/v1/sample/
+ - vllm/beam_search.py
+ - tests/samplers
+ - tests/conftest.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
- - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
- - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+ - pytest -v -s samplers
+#------------------------------------------------------------ mi300 · misc ------------------------------------------------------------#
-- label: Language Models Test (Extended Generation) # 32.2m
- timeout_in_minutes: 55
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Python-only Installation # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/language/generation
+ - tests/standalone_tests/python_only_compile.sh
+ - setup.py
+ - vllm/platforms/rocm.py
commands:
- - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
- - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
- - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
-
+ - bash standalone_tests/python_only_compile.sh
-- label: Multi-Modal Processor # 1h 42m
- timeout_in_minutes: 138
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
+- label: Regression # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/multimodal
- - tests/models/registry.py
+ - tests/test_regression
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/processing/test_tensor_schema.py
+ - pip install modelscope
+ - pytest -v -s test_regression.py
+#--------------------------------------------------------- mi300 · ray_compat ---------------------------------------------------------#
-- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
+- label: Ray Dependency Compatibility Check # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- torch_nightly: true
- optional: true
- working_dir: "/vllm-workspace/tests"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal
+ - requirements/
+ - setup.py
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
+ - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
+#------------------------------------------------------------ mi300 · v1 -------------------------------------------------------------#
-- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
+- label: Acceptance Length Test (Large Models) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- torch_nightly: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal
+ - vllm/v1/spec_decode/
+ - vllm/model_executor/models/mlp_speculator.py
+ - tests/v1/spec_decode/test_acceptance_length.py
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
-
+ - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
+ - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
-- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
+- label: e2e Core (1 GPU) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- torch_nightly: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
- - tests/models/multimodal/test_mapping.py
+ - vllm/v1/
+ - tests/v1/e2e/
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
-
+ - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
-- label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
+- label: e2e Scheduling (1 GPU) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- torch_nightly: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
+ - vllm/v1/
+ - tests/v1/e2e/general/
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
- - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
-
+ - pytest -v -s v1/e2e/general/test_async_scheduling.py
-- label: Multi-Modal Models (Extended Generation 1) # 1h 2m
- timeout_in_minutes: 106
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: Engine (1 GPU) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
- - tests/models/multimodal/test_mapping.py
+ - vllm/v1/engine/
+ - tests/v1/engine/
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- - pytest -v -s models/multimodal/test_mapping.py
-
+ - pytest -v -s v1/engine/test_preprocess_error_handling.py
+ - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
-- label: Multi-Modal Models (Extended Generation 2) # TBD
+- label: Spec Decode Draft Model # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
+ - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
+- label: Spec Decode Eagle # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
+ commands:
+ - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
-- label: Multi-Modal Models (Extended Generation 3) # TBD
+- label: Spec Decode Ngram + Suffix # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
-
+ - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
-- label: Multi-Modal Models (Extended Pooling) # TBD
+- label: Spec Decode Speculators + MTP # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/pooling
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s models/multimodal/pooling -m 'not core_model'
-
+ - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
-- label: Quantized Models Test # 21.4m
- timeout_in_minutes: 38
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: V1 attention (H100-MI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/layers/quantization
+ - vllm/config/attention.py
+ - vllm/model_executor/layers/attention
+ - vllm/v1/attention
+ - tests/v1/attention
- vllm/_aiter_ops.py
+ - vllm/envs.py
- vllm/platforms/rocm.py
- - tests/models/quantization
- - vllm/model_executor/model_loader/
commands:
- - pytest -v -s models/quantization
-
+ - pytest -v -s v1/attention
+
+- label: V1 Core + KV + Metrics # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/v1/core
+ - tests/v1/executor
+ - tests/v1/kv_offload
+ - tests/v1/worker
+ - tests/v1/kv_connector/unit
+ - tests/v1/metrics
+ - tests/entrypoints/openai/correctness/test_lmeval.py
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - pytest -v -s -m 'not cpu_test' v1/core
+ - pytest -v -s v1/executor
+ - pytest -v -s v1/kv_offload
+ - pytest -v -s v1/worker
+ - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
+ - pytest -v -s -m 'not cpu_test' v1/metrics
+ - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
+ # - export HSA_NO_SCRATCH_RECLAIM=1
+ - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-- label: Transformers Nightly Models # 50.9m
- timeout_in_minutes: 102
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
+- label: V1 others (CPU) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ no_gpu: true
optional: true
- working_dir: "/vllm-workspace/"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/multimodal/
- - vllm/model_executor/layers/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - tests/models/
- - examples/
+ - vllm/
+ - tests/v1
commands:
- - pip install --upgrade git+https://github.com/huggingface/transformers
- - pytest -v -s tests/models/test_initialization.py
- - pytest -v -s tests/models/test_transformers.py
- - pytest -v -s tests/models/multimodal/processing/
- - pytest -v -s tests/models/multimodal/test_mapping.py
- - python3 examples/basic/offline_inference/chat.py
- - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
- - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
-
+ - pytest -v -s -m 'cpu_test' v1/core
+ - pytest -v -s v1/structured_output
+ - pytest -v -s v1/test_serial_utils.py
+ - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
+ - pytest -v -s -m 'cpu_test' v1/metrics
-- label: Quantized MoE Test (B200-MI325) # TBD
+- label: V1 Sample + Logits # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_1
+ optional: true
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - tests/quantization/test_gfx3xx_moe.py
- - vllm/model_executor/models/deepseek_v2.py
- - vllm/model_executor/models/gpt_oss.py
- - vllm/model_executor/models/llama4.py
- - vllm/model_executor/layers/fused_moe
- - vllm/model_executor/layers/quantization/compressed_tensors
- - vllm/model_executor/layers/quantization/modelopt.py
- - vllm/model_executor/layers/quantization/mxfp4.py
- - vllm/v1/attention/backends/triton_attn.py
- - vllm/v1/attention/backends/rocm_attn.py
- - vllm/v1/attention/backends/rocm_aiter_fa.py
- - vllm/v1/attention/backends/mla/
- - vllm/v1/attention/selector.py
- - vllm/model_executor/layers/layernorm.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - vllm/model_executor/model_loader/
+ - vllm/
+ - tests/v1/sample
+ - tests/v1/logits_processors
+ - tests/v1/test_oracle.py
+ - tests/v1/test_request.py
+ - tests/v1/test_outputs.py
commands:
- - pytest -s -v tests/quantization/test_gfx3xx_moe.py
-
+ - pytest -v -s v1/sample
+ - pytest -v -s v1/logits_processors
+ - pytest -v -s v1/test_oracle.py
+ - pytest -v -s v1/test_request.py
+ - pytest -v -s v1/test_outputs.py
-- label: Distributed DP Tests (2 GPUs) # 56.1m
- timeout_in_minutes: 102
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
+- label: Distributed DP Tests (2 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
@@ -2451,152 +2286,126 @@ steps:
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
- DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
-
-- label: Distributed Compile + RPC Tests (2 GPUs) # 56.1m
- timeout_in_minutes: 102
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
+- label: Distributed Tests (2xH100-2xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
num_gpus: 2
- working_dir: "/vllm-workspace/tests"
+ working_dir: "/vllm-workspace/"
source_file_dependencies:
- - vllm/compilation/
- vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/worker/worker_base.py
- - vllm/v1/engine/
- - vllm/v1/worker/
- - tests/compile/fullgraph/test_basic_correctness.py
- - tests/compile/test_wrapper.py
- - tests/entrypoints/llm/test_collective_rpc.py
+ - vllm/v1/distributed/
+ - vllm/model_executor/layers/fused_moe/
+ - tests/v1/distributed/test_dbo.py
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s entrypoints/llm/test_collective_rpc.py
- - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
- - pytest -v -s ./compile/test_wrapper.py
-
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
+ - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
+ - pytest -v -s tests/v1/distributed/test_dbo.py
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
+ - pytest -v -s tests/distributed/test_packed_tensor.py
-- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # 56.1m
- timeout_in_minutes: 102
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
+- label: Metrics, Tracing (2 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/
- - vllm/engine/
- - vllm/executor/
- - vllm/worker/worker_base.py
- - vllm/v1/engine/
- - vllm/v1/worker/
- - tests/distributed/
- - tests/v1/shutdown
- - tests/v1/worker/test_worker_memory_snapshot.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/tracing
commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
- - pytest -v -s v1/worker/test_worker_memory_snapshot.py
-
+ - "pip install \
+ 'opentelemetry-sdk>=1.26.0' \
+ 'opentelemetry-api>=1.26.0' \
+ 'opentelemetry-exporter-otlp>=1.26.0' \
+ 'opentelemetry-semantic-conventions-ai>=0.4.1'"
+ - pytest -v -s v1/tracing
-- label: Distributed Model Tests (2 GPUs) # 19.3m
- timeout_in_minutes: 38
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
- num_gpus: 2
+- label: V1 e2e (2 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/model_loader/sharded_state_loader.py
- - vllm/model_executor/models/
- - vllm/model_executor/layers/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - tests/basic_correctness/
- - tests/model_executor/model_loader/test_sharded_state_loader.py
- - tests/models/
+ - vllm/
+ - tests/v1/e2e
commands:
- - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
- - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
- - pytest models/language -v -s -m 'distributed(num_gpus=2)'
- - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
- - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
-
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-- label: LoRA TP (Distributed) # 9.8m
- timeout_in_minutes: 18
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
+- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/lora
- - tests/lora
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
- - pytest -v -s -x lora/test_chatglm3_tp.py
- - pytest -v -s -x lora/test_llama_tp.py
- - pytest -v -s -x lora/test_llm_with_multi_loras.py
- - pytest -v -s -x lora/test_olmoe_tp.py
- - pytest -v -s -x lora/test_gptoss_tp.py
- - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - CROSS_LAYERS_BLOCKS=True ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-- label: Weight Loading Multiple GPU # 7.5m
- timeout_in_minutes: 14
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
- num_gpus: 2
+- label: Distributed DP Tests (4 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/weight_loading
+ - vllm/distributed/
+ - tests/v1/distributed
+ - tests/v1/engine/test_engine_core_client.py
+ - tests/distributed/test_utils
+ - vllm/platforms/rocm.py
commands:
- - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
+ - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
+ - pytest -v -s distributed/test_utils.py
-- label: Weight Loading Multiple GPU - Large Models # 12.6m
- timeout_in_minutes: 26
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
- num_gpus: 2
+- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/weight_loading
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ - vllm/platforms/rocm.py
commands:
- - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-- label: Ray Dependency Compatibility Check # TBD
+- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- optional: true
- working_dir: "/"
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ num_gpus: 4
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - requirements/
- - setup.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-- label: Distributed NixlConnector PD accuracy (4 GPUs) # 27.4m
- timeout_in_minutes: 44
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
+- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
num_gpus: 4
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
@@ -2604,161 +2413,160 @@ steps:
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+ - HYBRID_SSM=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
+- label: V1 e2e (4 GPUs) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ optional: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/v1/e2e
+ commands:
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
+
+- label: V1 e2e (4xH100-4xMI300) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_4
+ optional: true
+ source_file_dependencies:
+ - vllm/v1/attention/backends/utils.py
+ - vllm/v1/worker/gpu_model_runner.py
+ - tests/v1/e2e/test_hybrid_chunked_prefill.py
+ commands:
+ - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py
+#------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------#
-- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
+- label: Weight Loading Multiple GPU # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- optional: true
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/weight_loading
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - CROSS_LAYERS_BLOCKS=True ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
-- label: Distributed Tests (4 GPUs)(A100-MI325) # 20.9m
- timeout_in_minutes: 37
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
+- label: Weight Loading Multiple GPU - Large Models # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+ agent_pool: mi300_2
+ num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
+ - tests/weight_loading
commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s distributed/test_custom_all_reduce.py
- - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
- - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- - pytest -v -s -x lora/test_mixtral.py
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
+
+#########################################################################################################################################
+# #
+# MI325 (gfx942) tests #
+# #
+#########################################################################################################################################
+#---------------------------------------------------------- mi325 · compile ----------------------------------------------------------#
-- label: Distributed Tests (2 GPUs)(H100-MI325) # TBD
+- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_2
num_gpus: 2
- working_dir: "/vllm-workspace/"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
+ - vllm/compilation/
- vllm/distributed/
- - vllm/v1/distributed/
- - vllm/model_executor/layers/fused_moe/
- - tests/v1/distributed/test_dbo.py
- - vllm/_aiter_ops.py
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/worker/worker_base.py
+ - vllm/v1/engine/
+ - vllm/v1/worker/
+ - tests/compile/fullgraph/test_basic_correctness.py
+ - tests/compile/test_wrapper.py
+ - tests/entrypoints/llm/test_collective_rpc.py
- vllm/platforms/rocm.py
commands:
- export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s tests/v1/distributed/test_dbo.py
+ - pytest -v -s entrypoints/llm/test_collective_rpc.py
+ - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
+ - pytest -v -s ./compile/test_wrapper.py
+#-------------------------------------------------------- mi325 · distributed --------------------------------------------------------#
-- label: Distributed Compile Unit Tests (2xH100-2xMI325) # 14.3m
- timeout_in_minutes: 32
+- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_2
num_gpus: 2
- working_dir: "/vllm-workspace/"
- source_file_dependencies:
- - vllm/compilation/
- - vllm/model_executor/layers
- - tests/compile/passes/distributed/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - export VLLM_TEST_CLEAN_GPU_MEMORY=1
- - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
- - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
- # TODO: this test is not supported on ROCm, there are aiter kernels for this.
- # - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
- # - pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm
- # - "VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'"
-
-
-- label: LM Eval Small Models # 13.3m
- timeout_in_minutes: 23
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
-
-
-- label: LM Eval Small Models (MI325) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
- source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/
+ - vllm/engine/
+ - vllm/executor/
+ - vllm/worker/worker_base.py
+ - vllm/v1/engine/
+ - vllm/v1/worker/
+ - tests/distributed/
+ - tests/v1/shutdown
+ - tests/v1/worker/test_worker_memory_snapshot.py
- vllm/platforms/rocm.py
commands:
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+ - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
+ - pytest -v -s v1/worker/test_worker_memory_snapshot.py
-- label: LM Eval Small Models (B200-MI325) # TBD
+- label: Distributed Compile + Comm (4 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
+ agent_pool: mi325_4
+ num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/
+ - tests/distributed/test_pynccl
+ - tests/distributed/test_events
+ - tests/compile/fullgraph/test_basic_correctness.py
+ - tests/distributed/test_symm_mem_allreduce.py
+ - tests/distributed/test_multiproc_executor.py
- vllm/platforms/rocm.py
commands:
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s compile/fullgraph/test_basic_correctness.py
+ - pytest -v -s distributed/test_pynccl.py
+ - pytest -v -s distributed/test_events.py
+ - pytest -v -s distributed/test_symm_mem_allreduce.py
+ - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
+#---------------------------------------------------------- mi325 · engine -----------------------------------------------------------#
-- label: LM Eval Large Models (H200-MI325) # TBD
+- label: Engine # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_8
- optional: true
- num_gpus: 8
+ agent_pool: mi325_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/model_executor/layers/quantization/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/model_executor/layers/layernorm.py
- - csrc/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - tests/evals/
+ - vllm/
+ - tests/engine
+ - tests/test_sequence
+ - tests/test_config
+ - tests/test_logger
+ - tests/test_vllm_port
commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
+ - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
+#----------------------------------------------------------- mi325 · evals -----------------------------------------------------------#
-- label: LM Eval Large Models (4 GPUs)(FP8) # 24.8m
- timeout_in_minutes: 42
+- label: LM Eval Large Models (4xH100-4xMI325) # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_4
num_gpus: 4
@@ -2777,29 +2585,7 @@ steps:
- export VLLM_USE_DEEP_GEMM=0
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
-
-- label: LM Eval Large Models (4 GPUs)(A100-MI325) # 17.3m
- timeout_in_minutes: 27
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- optional: true
- working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
- source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- commands:
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
-
-
-- label: ROCm LM Eval Large Models (8 Card) # TBD
+- label: ROCm LM Eval Large Models (8 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_8
@@ -2820,232 +2606,143 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
+#---------------------------------------------------------- mi325 · models -----------------------------------------------------------#
-- label: GPQA Eval (GPT-OSS) (H100-MI325) # TBD
+- label: Language Models Test (Extended Generation) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_2
- num_gpus: 2
- optional: true
+ agent_pool: mi325_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/model_executor/layers/fused_moe/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - tests/evals/gpt_oss/
+ - vllm/
+ - tests/models/language/generation
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
-
+ - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
+ - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
-- label: DeepSeek V2-Lite Accuracy # 6.7m
- timeout_in_minutes: 12
+- label: Language Models Tests (Hybrid) %N # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- optional: true
- working_dir: "/vllm-workspace"
+ agent_pool: mi325_1
+ torch_nightly: true
+ parallelism: 2
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/distributed/eplb
- - vllm/model_executor/layers/fused_moe/
- - vllm/model_executor/layers/quantization/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/backends/mla/
- - vllm/v1/attention/selector.py
- - .buildkite/scripts/scheduled_integration_test/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/models/language/generation
commands:
- - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
-
+ - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
+ - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
-- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI325) # TBD
+- label: Multi-Modal Models (Extended Pooling) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_1
- num_gpus: 1
- working_dir: "/vllm-workspace"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/model_executor/layers/fused_moe/
- - vllm/model_executor/layers/quantization/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/backends/mla/
- - vllm/v1/attention/selector.py
- - .buildkite/scripts/scheduled_integration_test/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/models/multimodal/pooling
commands:
- - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
-
-
-- label: Qwen3-30B-A3B-FP8-block Accuracy # 6.4m
- timeout_in_minutes: 11
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- optional: true
- working_dir: "/vllm-workspace"
- source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/model_executor/layers/quantization/
- - vllm/distributed/eplb
- - vllm/model_executor/layers/fused_moe/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - .buildkite/scripts/scheduled_integration_test/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - pytest -v -s models/multimodal/pooling -m 'not core_model'
+
+- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
+ agent_pool: mi325_1
+ torch_nightly: true
+ working_dir: "/vllm-workspace/tests"
+ source_file_dependencies:
+ - vllm/
+ - tests/models/multimodal
commands:
- - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
+ - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
+#------------------------------------------------------------ mi325 · v1 -------------------------------------------------------------#
-- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # 10.9m
- timeout_in_minutes: 22
+- label: V1 Spec Decode # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_4
- num_gpus: 4
- optional: true
- working_dir: "/vllm-workspace"
+ agent_pool: mi325_1
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/spec_decode/
- - vllm/distributed/eplb
- - vllm/model_executor/layers/fused_moe/
- - vllm/model_executor/layers/quantization/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - .buildkite/scripts/scheduled_integration_test/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/spec_decode
commands:
- - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
-
-##### .buildkite/test_areas/compile.yaml #####
-# Slowly setting up the tests so that it is also easier for the
-# CI team to review and upstream to the pipelinev2.
-# The following tests are important for vLLM IR Ops refactoring,
-# which affects fusion passes on ROCm. So we have to
-# enable them as as soon as possible.
+ - pytest -v -s -m 'not slow_test' v1/spec_decode
-## TODO: Enable the test in this group
-# # corresponds to .buildkite/test_areas/compile.yaml
-# - label: Fusion and Compile Unit Tests (2xB200-2xMI325) # TBD
-# timeout_in_minutes: 180
-# mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325, tj]
-# agent_pool: mi325_1 # changed to 1 GPU until the fusion all reduce is enabled then only revert back to 2 GPUs
-# num_gpus: 1
-# working_dir: "/vllm-workspace/"
-# source_file_dependencies:
-# - csrc/quantization/fp4/
-# - vllm/model_executor/layers/quantization/
-# - vllm/model_executor/layers/layernorm.py
-# - vllm/model_executor/layers/activation.py
-# - vllm/model_executor/layers/attention/attention.py
-# - vllm/v1/attention/backends/flashinfer.py
-# - vllm/compilation/ # TODO(luka) limit to vllm/compilation/passes
-# - tests/compile/test_fusion_attn.py
-# - tests/compile/test_silu_mul_quant_fusion.py
-# - tests/compile/distributed/test_fusion_all_reduce.py
-# - tests/compile/fullgraph/test_full_graph.py
-# commands:
-# - rocm-smi
-# # we run all backend tests on ROCm
-# # These two tests are covered in "PyTorch Compilation Passes Unit Tests"
-# # - "pytest -v -s tests/compile/passes/test_fusion_attn.py"
-# # - "pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py"
-# # TODO: this test is not supported on ROCm, there are aiter kernels for this.
-# # - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
-# # TODO: find out more details
-# # - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile
+#########################################################################################################################################
+# #
+# MI355 (gfx950) tests #
+# #
+#########################################################################################################################################
+#-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
-- label: Fusion E2E Quick (H100-MI325) # TBD
+- label: Attention Benchmarks Smoke Test (B200-MI355) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- num_gpus: 1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+ agent_pool: mi355_2
+ num_gpus: 2
working_dir: "/vllm-workspace/"
source_file_dependencies:
- - csrc/quantization/
- - vllm/model_executor/
+ - benchmarks/attention_benchmarks/
- vllm/v1/attention/
- - vllm/compilation/
- - tests/compile/fusions_e2e/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - rocm-smi
- # Run all models and attn backends but only Inductor partition and native custom ops
- - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
- # Different from CUDA, Qwen requires +rms_norm and +quant_fp8 as rms+quant fusion is only supported on AITER
- - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
+ - python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1
+#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
-- label: Fusion E2E Config Sweep (H100-MI325) # TBD
+- label: Distributed Tests (2xH100-2xMI355) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
- agent_pool: mi325_1
- num_gpus: 1
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+ agent_pool: mi355_2
+ num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/"
source_file_dependencies:
- - csrc/quantization/
- - vllm/compilation/
- - vllm/model_executor/layers/layernorm.py
- - vllm/model_executor/layers/activation.py
- - vllm/model_executor/layers/attention/attention.py
- - vllm/model_executor/layers/quantization/input_quant_fp8.py
- - tests/compile/fusions_e2e/
+ - vllm/distributed/
+ - vllm/v1/distributed/
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - tests/distributed/test_context_parallel.py
+ - tests/v1/distributed/test_dbo.py
+ - examples/offline_inference/data_parallel.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - rocm-smi
- - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
-
-## There are no ops on ROCm for these tests.
-## The test still passes but the logs are not useful.
-## fused ops just call torch.ops.symm_mem which
-## exists in ROCm even though they don't work
-# - label: AsyncTP Correctness Tests (2xH100-2xMI325)
-# - label: Fusion E2E TP2 Quick (H100-MI325)
-# - label: Fusion E2E TP2 AsyncTP Config Sweep (H100-MI325)
-# - label: Fusion E2E TP2 (B200-MI325)
-# - label: Sequence Parallel Correctness Tests (2xH100-2xMI325)
-
+ - export TORCH_NCCL_BLOCKING_WAIT=1
+ - pytest -v -s tests/distributed/test_context_parallel.py
+ - pytest -v -s tests/v1/distributed/test_dbo.py
-#####################################################################################################################################
-# #
-# gfx950 #
-# #
-#####################################################################################################################################
+#-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
-- label: Entrypoints Integration (API Server openai - Part 1) # TBD
+- label: Entrypoints Integration (API Server 2) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
+ optional: true
fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/entrypoints/openai
- - tests/entrypoints/test_chat_utils
+ - tests/entrypoints/rpc
+ - tests/entrypoints/serve/instrumentator
+ - tests/tool_use
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
-
+ - pytest -v -s entrypoints/serve/instrumentator
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
+ - pytest -v -s tool_use
-- label: Entrypoints Integration (API Server openai - Part 2) # TBD
+- label: Entrypoints Integration (API Server openai - Part 1) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
@@ -3058,12 +2755,9 @@ steps:
- tests/entrypoints/test_chat_utils
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
- - pytest -v -s entrypoints/openai/speech_to_text/
- - pytest -v -s entrypoints/test_chat_utils.py
-
+ - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
-- label: Entrypoints Integration (API Server openai - Part 3) # TBD
+- label: Entrypoints Integration (API Server openai - Part 2) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
@@ -3076,28 +2770,24 @@ steps:
- tests/entrypoints/test_chat_utils
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
-
+ - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
+ - pytest -v -s entrypoints/openai/speech_to_text/
+ - pytest -v -s entrypoints/test_chat_utils.py
-- label: Entrypoints Integration (API Server 2) # TBD
+- label: Entrypoints Integration (API Server openai - Part 3) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
- optional: true
fast_check: true
torch_nightly: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/entrypoints/rpc
- - tests/entrypoints/serve/instrumentator
- - tests/tool_use
+ - tests/entrypoints/openai
+ - tests/entrypoints/test_chat_utils
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- - pytest -v -s entrypoints/serve/instrumentator
- - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
- - pytest -v -s tool_use
-
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
- label: Entrypoints Integration (Pooling) # TBD
timeout_in_minutes: 180
@@ -3113,117 +2803,112 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/pooling
+#----------------------------------------------------------- mi355 · evals -----------------------------------------------------------#
-- label: Regression # TBD
+- label: GPQA Eval (GPT-OSS) (2xB200-2xMI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
+ agent_pool: mi355_2
+ num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/test_regression
- commands:
- - pip install modelscope
- - pytest -v -s test_regression.py
-
-
-- label: V1 Spec Decode # TBD
- timeout_in_minutes: 60
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/v1/spec_decode
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/model_executor/layers/fused_moe/
+ - tests/evals/gpt_oss/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s -m 'not slow_test' v1/spec_decode
-
+ - uv pip install --system 'gpt-oss[eval]==0.0.5'
+ - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
-- label: V1 Sample + Logits # TBD
- timeout_in_minutes: 60
+- label: LM Eval Qwen3-5 Models (B200-MI355) # TBD
+ timeout_in_minutes: 120
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
+ agent_pool: mi355_2
+ num_gpus: 2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1/sample
- - tests/v1/logits_processors
- - tests/v1/test_oracle.py
- - tests/v1/test_request.py
- - tests/v1/test_outputs.py
+ - vllm/model_executor/models/qwen3_5.py
+ - vllm/model_executor/models/qwen3_5_mtp.py
+ - vllm/transformers_utils/configs/qwen3_5.py
+ - vllm/transformers_utils/configs/qwen3_5_moe.py
+ - vllm/model_executor/models/qwen.py
+ - vllm/model_executor/models/qwen2.py
+ - vllm/model_executor/models/qwen3.py
+ - vllm/model_executor/models/qwen3_next.py
+ - vllm/model_executor/models/qwen3_next_mtp.py
+ - vllm/model_executor/layers/fla/ops/
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/sample
- - pytest -v -s v1/logits_processors
- - pytest -v -s v1/test_oracle.py
- - pytest -v -s v1/test_request.py
- - pytest -v -s v1/test_outputs.py
-
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
-- label: V1 Core + KV + Metrics # TBD
- timeout_in_minutes: 60
+- label: LM Eval Small Models (2xB200-2xMI355) # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
+ agent_pool: mi355_2
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/v1/core
- - tests/v1/executor
- - tests/v1/kv_offload
- - tests/v1/worker
- - tests/v1/kv_connector/unit
- - tests/v1/metrics
- - tests/entrypoints/openai/correctness/test_lmeval.py
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - pytest -v -s -m 'not cpu_test' v1/core
- - pytest -v -s v1/executor
- - pytest -v -s v1/kv_offload
- - pytest -v -s v1/worker
- - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
- - pytest -v -s -m 'not cpu_test' v1/metrics
- - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
- - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
-- label: V1 Speculative Decoding (slow) # TBD
+- label: Qwen3-30B-A3B-FP8-block Accuracy (B200-MI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- working_dir: "/vllm-workspace/tests"
+ agent_pool: mi355_2
+ num_gpus: 2
+ working_dir: "/vllm-workspace"
source_file_dependencies:
- - vllm/v1/spec_decode/
- vllm/model_executor/models/
- - vllm/v1/attention/
- - vllm/model_executor/layers/
- - tests/v1/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/model_executor/layers/quantization/
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/distributed/eplb
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
+ - .buildkite/scripts/scheduled_integration_test/
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_eagle.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_extract_hidden_states.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_max_len.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_mtp.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_ngram.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_speculators_eagle3.py
- - pytest -v -s -m 'slow_test' v1/spec_decode/test_tree_attention.py
-
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
-- label: V1 attention (B200-MI355) # TBD
+- label: LM Eval Large Models (4xH100-4xMI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- working_dir: "/vllm-workspace/tests"
+ agent_pool: mi355_4
+ num_gpus: 4
+ optional: true
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- - vllm/config/attention.py
- - vllm/model_executor/layers/attention
- - vllm/v1/attention
- - tests/v1/attention
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - vllm/model_executor/models/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/attention/backends/
+ - vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- - vllm/envs.py
- vllm/platforms/rocm.py
commands:
- - pytest -v -s v1/attention
+ - export VLLM_USE_DEEP_GEMM=0
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
+#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
- label: Examples # TBD
timeout_in_minutes: 180
@@ -3258,6 +2943,31 @@ steps:
- python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
- python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
+#---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------#
+
+- label: Kernels (B200-MI355) # TBD
+ timeout_in_minutes: 180
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+ agent_pool: mi355_1
+ working_dir: "/vllm-workspace/"
+ source_file_dependencies:
+ - csrc/quantization/fp4/
+ - csrc/attention/mla/
+ - csrc/quantization/cutlass_w8a8/moe/
+ - vllm/model_executor/layers/fused_moe/cutlass_moe.py
+ - vllm/v1/attention/backends/triton_attn.py
+ - vllm/v1/attention/backends/rocm_attn.py
+ - vllm/v1/attention/backends/rocm_aiter_fa.py
+ - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
+ - vllm/v1/attention/backends/mla/aiter_triton_mla.py
+ - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
+ - vllm/v1/attention/selector.py
+ - vllm/platforms/rocm.py
+ - vllm/_aiter_ops.py
+ commands:
+ - rocm-smi
+ - python3 examples/basic/offline_inference/chat.py
+ - pytest -v -s tests/kernels/attention/test_attention_selector.py
- label: Kernels Attention Test %N # TBD
timeout_in_minutes: 180
@@ -3277,25 +2987,6 @@ steps:
commands:
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-
-- label: Kernels Quantization Test %N # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- parallelism: 2
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - csrc/quantization/
- - vllm/model_executor/layers/quantization
- - tests/kernels/quantization
- - tests/kernels/quantization/test_rocm_skinny_gemms.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - vllm/model_executor/kernels/
- commands:
- - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-
-
- label: Kernels MoE Test %N # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3316,54 +3007,40 @@ steps:
- pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-
-- label: Kernels FP8 MoE Test # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - csrc/moe/
- - csrc/quantization/w8a8/cutlass/moe/
- - vllm/model_executor/layers/fused_moe/
- - tests/kernels/moe/test_deepep_moe.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - vllm/envs.py
- commands:
- - pytest -v -s kernels/moe/test_deepep_moe.py
-
-
-- label: Quantization # TBD
+- label: Kernels Quantization Test %N # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
+ parallelism: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
+ - csrc/quantization/
- vllm/model_executor/layers/quantization
- - tests/quantization
+ - tests/kernels/quantization
+ - tests/kernels/quantization/test_rocm_skinny_gemms.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+ - vllm/model_executor/kernels/
commands:
- - uv pip install --system torchao==0.17.0
- - uv pip install --system conch-triton-kernels
- - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
-
+ - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-- label: Language Models Tests (Standard) # TBD
+- label: Kernels FP8 MoE Test (2xH100-2xMI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- torch_nightly: true
+ agent_pool: mi355_2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/language
+ - csrc/moe/
+ - csrc/quantization/w8a8/cutlass/moe/
+ - vllm/model_executor/layers/fused_moe/
+ - tests/kernels/moe/test_deepep_moe.py
+ - vllm/_aiter_ops.py
+ - vllm/platforms/rocm.py
+ - vllm/envs.py
commands:
- - pip freeze | grep -E 'torch'
- - pytest -v -s models/language -m 'core_model and (not slow_test)'
+ - pytest -v -s kernels/moe/test_deepep_moe.py
+#---------------------------------------------------------- mi355 · models -----------------------------------------------------------#
- label: Language Models Test (Extended Generation) # TBD
timeout_in_minutes: 180
@@ -3375,10 +3052,9 @@ steps:
- tests/models/language/generation
commands:
- uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
- - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
-
- label: Language Models Test (Extended Pooling) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3390,71 +3066,46 @@ steps:
commands:
- pytest -v -s models/language/pooling -m 'not core_model'
-
-- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- torch_nightly: true
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
-
-
-- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- torch_nightly: true
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
-
-
-- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
+- label: Language Models Test (PPL) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
- torch_nightly: true
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
- - tests/models/multimodal/test_mapping.py
+ - vllm/model_executor/models/qwen3_5.py
+ - vllm/model_executor/models/qwen3_5_mtp.py
+ - vllm/transformers_utils/configs/qwen3_5.py
+ - vllm/transformers_utils/configs/qwen3_5_moe.py
+ - vllm/model_executor/models/qwen.py
+ - vllm/model_executor/models/qwen2.py
+ - vllm/model_executor/models/qwen3.py
+ - vllm/model_executor/models/qwen3_next.py
+ - vllm/model_executor/models/qwen3_next_mtp.py
+ - vllm/model_executor/layers/fla/ops/
+ - vllm/_aiter_ops.py
+ - vllm/v1/attention/backends/triton_attn.py
+ - vllm/v1/attention/backends/rocm_attn.py
+ - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
+ - vllm/v1/attention/backends/rocm_aiter_fa.py
+ - vllm/v1/attention/backends/flex_attention.py
+ - vllm/v1/attention/ops/
+ - vllm/platforms/rocm.py
+ - tests/models/language/generation_ppl_test
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
-
+ - pytest -v -s models/language/generation_ppl_test
-- label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
+- label: Language Models Tests (Standard) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
torch_nightly: true
- optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/models/multimodal/generation
+ - tests/models/language
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
- - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
-
+ - pip freeze | grep -E 'torch'
+ - pytest -v -s models/language -m 'core_model and (not slow_test)'
- label: Multi-Modal Models (Extended Generation 1) # TBD
timeout_in_minutes: 180
@@ -3471,21 +3122,6 @@ steps:
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- pytest -v -s models/multimodal/test_mapping.py
-
-- label: Multi-Modal Models (Extended Generation 2) # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- optional: true
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/
- - tests/models/multimodal/generation
- commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
-
-
- label: Multi-Modal Models (Extended Generation 3) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3499,7 +3135,6 @@ steps:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
-
- label: Multi-Modal Models (Extended Pooling) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3512,303 +3147,252 @@ steps:
commands:
- pytest -v -s models/multimodal/pooling -m 'not core_model'
-
-- label: Quantized Models Test # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_1
- working_dir: "/vllm-workspace/tests"
- source_file_dependencies:
- - vllm/model_executor/layers/quantization
- - tests/models/quantization
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
- - vllm/model_executor/model_loader/
- commands:
- - pytest -v -s models/quantization
-
-
-- label: Kernels (B200-MI355) # TBD
+- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
- working_dir: "/vllm-workspace/"
- source_file_dependencies:
- - csrc/quantization/fp4/
- - csrc/attention/mla/
- - csrc/quantization/cutlass_w8a8/moe/
- - vllm/model_executor/layers/fused_moe/cutlass_moe.py
- - vllm/v1/attention/backends/triton_attn.py
- - vllm/v1/attention/backends/rocm_attn.py
- - vllm/v1/attention/backends/rocm_aiter_fa.py
- - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- - vllm/v1/attention/backends/mla/aiter_triton_mla.py
- - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
- - vllm/v1/attention/selector.py
- - vllm/platforms/rocm.py
- - vllm/_aiter_ops.py
- commands:
- - rocm-smi
- - python3 examples/basic/offline_inference/chat.py
- - pytest -v -s tests/kernels/attention/test_attention_selector.py
-
-
-- label: Weight Loading Multiple GPU # TBD
- timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
+ torch_nightly: true
+ optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/weight_loading
+ - tests/models/multimodal
commands:
- - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
-
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
+ - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
-- label: Weight Loading Multiple GPU - Large Models # TBD
+- label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- working_dir: "/vllm-workspace/tests"
- num_gpus: 2
+ agent_pool: mi355_1
+ torch_nightly: true
optional: true
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- - tests/weight_loading
+ - tests/models/multimodal/generation
commands:
- - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
-
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
+ - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
+ - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
-- label: Ray Dependency Compatibility Check # TBD
+- label: Quantized Models Test # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_1
- optional: true
- working_dir: "/"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - requirements/
- - setup.py
+ - vllm/model_executor/layers/quantization
+ - tests/models/quantization
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+ - vllm/model_executor/model_loader/
commands:
- - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
+ - pytest -v -s models/quantization
+#------------------------------------------------------- mi355 · quantization --------------------------------------------------------#
-- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
+- label: Quantization # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_4
- num_gpus: 4
- optional: true
+ agent_pool: mi355_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
+ - csrc/
+ - vllm/model_executor/layers/quantization
+ - tests/quantization
+ - vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+ - uv pip install --system torchao==0.17.0
+ - uv pip install --system conch-triton-kernels
+ - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
+
+# - label: Quantized MoE Test (B200-MI355) # TBD
+# timeout_in_minutes: 180
+# mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+# agent_pool: mi355_1
+# working_dir: "/vllm-workspace/"
+# source_file_dependencies:
+# - tests/quantization/test_gfx950_moe.py
+# - vllm/model_executor/models/deepseek_v2.py
+# - vllm/model_executor/models/gpt_oss.py
+# - vllm/model_executor/models/llama4.py
+# - vllm/model_executor/layers/fused_moe
+# - vllm/model_executor/layers/quantization/compressed_tensors
+# - vllm/model_executor/layers/quantization/modelopt.py
+# - vllm/model_executor/layers/quantization/mxfp4.py
+# - vllm/v1/attention/backends/triton_attn.py
+# - vllm/v1/attention/backends/rocm_attn.py
+# - vllm/v1/attention/backends/rocm_aiter_fa.py
+# - vllm/v1/attention/backends/mla/
+# - vllm/v1/attention/selector.py
+# - vllm/model_executor/layers/layernorm.py
+# - vllm/_aiter_ops.py
+# - vllm/platforms/rocm.py
+# - vllm/model_executor/model_loader/
+# commands:
+# - pytest -s -v tests/quantization/test_gfx950_moe.py
+#------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------#
-- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
+- label: V1 attention (B200-MI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_4
- num_gpus: 4
- optional: true
+ agent_pool: mi355_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
+ - vllm/config/attention.py
+ - vllm/model_executor/layers/attention
+ - vllm/v1/attention
+ - tests/v1/attention
+ - vllm/_aiter_ops.py
+ - vllm/envs.py
- vllm/platforms/rocm.py
commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
+ - pytest -v -s v1/attention
-- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD
- timeout_in_minutes: 180
+- label: V1 Core + KV + Metrics # TBD
+ timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
- optional: true
+ agent_pool: mi355_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - vllm/v1/worker/kv_connector_model_runner_mixin.py
- - tests/v1/kv_connector/nixl_integration/
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/core
+ - tests/v1/executor
+ - tests/v1/kv_offload
+ - tests/v1/worker
+ - tests/v1/kv_connector/unit
+ - tests/v1/metrics
+ - tests/entrypoints/openai/correctness/test_lmeval.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
-
+ - pytest -v -s -m 'not cpu_test' v1/core
+ - pytest -v -s v1/executor
+ - pytest -v -s v1/kv_offload
+ - pytest -v -s v1/worker
+ - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
+ - pytest -v -s -m 'not cpu_test' v1/metrics
+ - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
+ - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-- label: Distributed Tests (2 GPUs)(H100-MI355) # TBD
- timeout_in_minutes: 180
+- label: V1 Sample + Logits # TBD
+ timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
- optional: true
- working_dir: "/vllm-workspace/"
+ agent_pool: mi355_1
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/distributed/
- - vllm/v1/distributed/
- - vllm/model_executor/layers/fused_moe/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - tests/distributed/test_context_parallel.py
- - tests/v1/distributed/test_dbo.py
- - examples/offline_inference/data_parallel.py
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/sample
+ - tests/v1/logits_processors
+ - tests/v1/test_oracle.py
+ - tests/v1/test_request.py
+ - tests/v1/test_outputs.py
commands:
- - export TORCH_NCCL_BLOCKING_WAIT=1
- - pytest -v -s tests/distributed/test_context_parallel.py
- - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- - pytest -v -s tests/v1/distributed/test_dbo.py
-
+ - pytest -v -s v1/sample
+ - pytest -v -s v1/logits_processors
+ - pytest -v -s v1/test_oracle.py
+ - pytest -v -s v1/test_request.py
+ - pytest -v -s v1/test_outputs.py
-- label: Distributed Compile Unit Tests (2xH100-2xMI355) # TBD
- timeout_in_minutes: 180
+- label: V1 Spec Decode # TBD
+ timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
- optional: true
- working_dir: "/vllm-workspace/"
+ agent_pool: mi355_1
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/compilation/
- - vllm/model_executor/layers
- - tests/compile/passes/distributed/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/v1/spec_decode
commands:
- - export VLLM_TEST_CLEAN_GPU_MEMORY=1
- - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
- - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
- # TODO: this test is not supported on ROCm, there are aiter kernels for this.
- # - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
- # - pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm
- # - "VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'"
-
+ - pytest -v -s -m 'not slow_test' v1/spec_decode
-- label: LM Eval Small Models (B200-MI355) # TBD
+- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_2
+ num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - vllm/v1/worker/kv_connector_model_runner_mixin.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
-- label: LM Eval Large Models (4 GPUs)(FP8) # TBD
+- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_4
num_gpus: 4
optional: true
- working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/_aiter_ops.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - export VLLM_USE_DEEP_GEMM=0
- - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
-
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-- label: GPQA Eval (GPT-OSS) (B200-MI355) # TBD
+- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
timeout_in_minutes: 180
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx955nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+ agent_pool: mi355_4
+ num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - csrc/
- - vllm/model_executor/layers/quantization
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - vllm/model_executor/layers/fused_moe/
- - tests/evals/gpt_oss/
- - vllm/_aiter_ops.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
+ - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+#------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------#
-- label: Qwen3-30B-A3B-FP8-block Accuracy (B200-MI355) # TBD
+- label: Weight Loading Multiple GPU # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_2
num_gpus: 2
- working_dir: "/vllm-workspace"
+ working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/
- - vllm/model_executor/model_loader/
- - vllm/model_executor/layers/quantization/
- - vllm/model_executor/layers/fused_moe/
- - vllm/distributed/eplb
- - vllm/v1/attention/backends/
- - vllm/v1/attention/selector.py
- - .buildkite/scripts/scheduled_integration_test/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/weight_loading
commands:
- - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
-
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
-- label: Attention Benchmarks Smoke Test (B200-MI355) # TBD
+- label: Weight Loading Multiple GPU - Large Models # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_2
+ working_dir: "/vllm-workspace/tests"
num_gpus: 2
- working_dir: "/vllm-workspace/"
+ optional: true
source_file_dependencies:
- - benchmarks/attention_benchmarks/
- - vllm/v1/attention/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/weight_loading
commands:
- - python3 benchmarks/attention_benchmarks/benchmark.py --backends ROCM_ATTN ROCM_AITER_FA ROCM_AITER_UNIFIED_ATTN --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
+#----------------------------------------------------------- mi355 · misc ------------------------------------------------------------#
-- label: LM Eval Qwen3-5 Models (B200-MI355) # TBD
- timeout_in_minutes: 120
+- label: Regression # TBD
+ timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
- agent_pool: mi355_2
- num_gpus: 2
+ agent_pool: mi355_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- - vllm/model_executor/models/qwen3_5.py
- - vllm/model_executor/models/qwen3_5_mtp.py
- - vllm/transformers_utils/configs/qwen3_5.py
- - vllm/transformers_utils/configs/qwen3_5_moe.py
- - vllm/model_executor/models/qwen.py
- - vllm/model_executor/models/qwen2.py
- - vllm/model_executor/models/qwen3.py
- - vllm/model_executor/models/qwen3_next.py
- - vllm/model_executor/models/qwen3_next_mtp.py
- - vllm/model_executor/layers/fla/ops/
- - vllm/_aiter_ops.py
- - vllm/platforms/rocm.py
+ - vllm/
+ - tests/test_regression
commands:
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
+ - pip install modelscope
+ - pytest -v -s test_regression.py
diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml
new file mode 100644
index 000000000000..a10fda41ef0d
--- /dev/null
+++ b/.buildkite/test_areas/disaggregated.yaml
@@ -0,0 +1,98 @@
+group: Disaggregated
+depends_on:
+ - image-build
+steps:
+- label: Distributed NixlConnector PD accuracy (4 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 4
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+- label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 4
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
+- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 4
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
+- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 4
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
+- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
+ timeout_in_minutes: 20
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 4
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
+- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 2
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
+
+- label: NixlConnector PD + Spec Decode acceptance (2 GPUs)
+ timeout_in_minutes: 30
+ device: a100
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 2
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - vllm/v1/worker/kv_connector_model_runner_mixin.py
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
+
+- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
+ timeout_in_minutes: 30
+ working_dir: "/vllm-workspace/tests"
+ num_devices: 2
+ source_file_dependencies:
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
+ - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
+ - tests/v1/kv_connector/nixl_integration/
+ commands:
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+ - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
\ No newline at end of file
diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml
index 19815e14b4e0..093f3ab4fe1f 100644
--- a/.buildkite/test_areas/distributed.yaml
+++ b/.buildkite/test_areas/distributed.yaml
@@ -196,6 +196,8 @@ steps:
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
- VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- pytest -v -s tests/v1/distributed/test_dbo.py
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
+ - pytest -v -s tests/distributed/test_packed_tensor.py
- label: Distributed Tests (2 GPUs)(B200)
device: b200
@@ -224,63 +226,6 @@ steps:
commands:
- ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code"
-- label: Distributed NixlConnector PD accuracy (4 GPUs)
- timeout_in_minutes: 30
- working_dir: "/vllm-workspace/tests"
- num_devices: 4
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
- - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs)
- timeout_in_minutes: 30
- working_dir: "/vllm-workspace/tests"
- num_devices: 4
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
- - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
- timeout_in_minutes: 30
- working_dir: "/vllm-workspace/tests"
- num_devices: 4
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
- - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs)
- timeout_in_minutes: 20
- working_dir: "/vllm-workspace/tests"
- num_devices: 4
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - tests/v1/kv_connector/nixl_integration/
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
- - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-- label: NixlConnector PD + Spec Decode acceptance (2 GPUs)
- timeout_in_minutes: 30
- device: a100
- working_dir: "/vllm-workspace/tests"
- num_devices: 2
- source_file_dependencies:
- - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
- - vllm/v1/worker/kv_connector_model_runner_mixin.py
- - tests/v1/kv_connector/nixl_integration/
- commands:
- - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
- - bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
-
- label: Pipeline + Context Parallelism (4 GPUs)
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml
index 5b7f96bc7a26..857fefd268a4 100644
--- a/.buildkite/test_areas/e2e_integration.yaml
+++ b/.buildkite/test_areas/e2e_integration.yaml
@@ -29,6 +29,15 @@ steps:
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
+- label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy
+ timeout_in_minutes: 60
+ device: h100
+ optional: true
+ num_devices: 4
+ working_dir: "/vllm-workspace"
+ commands:
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
+
- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100)
timeout_in_minutes: 60
device: h100
diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml
index a05ee886f5b9..2df0a5e253fb 100644
--- a/.buildkite/test_areas/kernels.yaml
+++ b/.buildkite/test_areas/kernels.yaml
@@ -18,10 +18,22 @@ steps:
source_file_dependencies:
- csrc/
- tests/kernels/core
- - tests/kernels/test_top_k_per_row.py
- tests/kernels/test_concat_mla_q.py
commands:
- - pytest -v -s kernels/core kernels/test_top_k_per_row.py kernels/test_concat_mla_q.py
+ - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py
+
+- label: Kernels MiniMax Reduce RMS Test (2 GPUs)
+ timeout_in_minutes: 15
+ num_devices: 2
+ device: h100
+ source_file_dependencies:
+ - csrc/minimax_reduce_rms_kernel.cu
+ - csrc/minimax_reduce_rms_kernel.h
+ - vllm/model_executor/layers/mamba/linear_attn.py
+ - vllm/model_executor/layers/mamba/lamport_workspace.py
+ - tests/kernels/core/test_minimax_reduce_rms.py
+ commands:
+ - pytest -v -s kernels/core/test_minimax_reduce_rms.py
- label: Kernels Attention Test %N
timeout_in_minutes: 35
@@ -107,6 +119,7 @@ steps:
- vllm/v1/attention/backends/mla/flashinfer_mla.py
- vllm/v1/attention/selector.py
- vllm/platforms/cuda.py
+ - tests/kernels/test_top_k_per_row.py
commands:
- nvidia-smi
- python3 examples/basic/offline_inference/chat.py
@@ -117,6 +130,7 @@ steps:
- pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py
- pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
- pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py
+ - pytest -v -s tests/kernels/test_top_k_per_row.py
# Quantization
- pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
- pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
@@ -127,6 +141,7 @@ steps:
- pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py
- pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
- pytest -v -s tests/kernels/moe/test_nvfp4_moe.py
+ - pytest -v -s tests/kernels/moe/test_mxfp4_moe.py
- pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py
- pytest -v -s tests/kernels/moe/test_flashinfer.py
- pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
@@ -141,7 +156,7 @@ steps:
- vllm/utils/import_utils.py
- tests/kernels/helion/
commands:
- - pip install helion==0.3.3
+ - pip install helion==1.0.0
- pytest -v -s kernels/helion/
@@ -186,7 +201,14 @@ steps:
timeout_in_minutes: 90
device: h100
num_devices: 2
- optional: true
+ source_file_dependencies:
+ - csrc/quantization/cutlass_w8a8/moe/
+ - csrc/moe/
+ - tests/kernels/moe
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/model_executor/layers/quantization/
+ - vllm/distributed/device_communicators/
+ - vllm/config
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
@@ -195,6 +217,13 @@ steps:
timeout_in_minutes: 90
device: b200
num_devices: 2
- optional: true
+ source_file_dependencies:
+ - csrc/quantization/cutlass_w8a8/moe/
+ - csrc/moe/
+ - tests/kernels/moe
+ - vllm/model_executor/layers/fused_moe/
+ - vllm/model_executor/layers/quantization/
+ - vllm/distributed/device_communicators/
+ - vllm/config
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml
index 39029efe9cd9..a07d702cf3ce 100644
--- a/.buildkite/test_areas/lm_eval.yaml
+++ b/.buildkite/test_areas/lm_eval.yaml
@@ -91,6 +91,16 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
+- label: LM Eval TurboQuant KV Cache
+ timeout_in_minutes: 75
+ source_file_dependencies:
+ - vllm/model_executor/layers/quantization/turboquant/
+ - vllm/v1/attention/backends/turboquant_attn.py
+ - vllm/v1/attention/ops/triton_turboquant_decode.py
+ - vllm/v1/attention/ops/triton_turboquant_store.py
+ commands:
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
+
- label: GPQA Eval (GPT-OSS) (H100)
timeout_in_minutes: 120
device: h100
diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml
index 82fe6670a990..0cf9ec433925 100644
--- a/.buildkite/test_areas/misc.yaml
+++ b/.buildkite/test_areas/misc.yaml
@@ -224,6 +224,7 @@ steps:
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
+ - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
- label: Acceptance Length Test (Large Models) # optional
timeout_in_minutes: 25
diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml
index 8ba4484fb02e..ed782c061fa3 100644
--- a/.buildkite/test_areas/models_basic.yaml
+++ b/.buildkite/test_areas/models_basic.yaml
@@ -1,10 +1,9 @@
group: Models - Basic
-depends_on:
+depends_on:
- image-build
steps:
- label: Basic Models Tests (Initialization)
timeout_in_minutes: 45
- device: h200_18gb
torch_nightly: true
source_file_dependencies:
- vllm/
@@ -13,10 +12,11 @@ steps:
commands:
# Run a subset of model initialization tests
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
+ mirror:
+ torch_nightly: {}
- label: Basic Models Tests (Extra Initialization) %N
timeout_in_minutes: 45
- torch_nightly: true
source_file_dependencies:
- vllm/model_executor/models/
- tests/models/test_initialization.py
@@ -27,6 +27,8 @@ steps:
# test.) Also run if model initialization test file is modified
- pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 2
+ mirror:
+ torch_nightly: {}
- label: Basic Models Tests (Other)
timeout_in_minutes: 45
@@ -42,10 +44,10 @@ steps:
device: mi325_1
depends_on:
- image-build-amd
-
+
- label: Basic Models Test (Other CPU) # 5min
- depends_on:
+ depends_on:
- image-build-cpu
timeout_in_minutes: 10
source_file_dependencies:
@@ -70,3 +72,18 @@ steps:
- python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
# Whisper needs spawn method to avoid deadlock
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
+
+- label: Transformers Backward Compatibility Models Test
+ working_dir: "/vllm-workspace/"
+ optional: true
+ soft_fail: true
+ commands:
+ - pip install transformers==4.57.5
+ - pytest -v -s tests/models/test_initialization.py
+ - pytest -v -s tests/models/test_transformers.py
+ - pytest -v -s tests/models/multimodal/processing/
+ - pytest -v -s tests/models/multimodal/test_mapping.py
+ - python3 examples/offline_inference/basic/chat.py
+ - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
+ # Whisper needs spawn method to avoid deadlock
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml
index 7eac9e30193a..c13371e25f1d 100644
--- a/.buildkite/test_areas/models_language.yaml
+++ b/.buildkite/test_areas/models_language.yaml
@@ -1,10 +1,9 @@
group: Models - Language
-depends_on:
+depends_on:
- image-build
steps:
- label: Language Models Tests (Standard)
timeout_in_minutes: 25
- torch_nightly: true
source_file_dependencies:
- vllm/
- tests/models/language
@@ -12,10 +11,11 @@ steps:
# Test standard language models, excluding a subset of slow tests
- pip freeze | grep -E 'torch'
- pytest -v -s models/language -m 'core_model and (not slow_test)'
+ mirror:
+ torch_nightly: {}
- label: Language Models Tests (Extra Standard) %N
timeout_in_minutes: 45
- torch_nightly: true
source_file_dependencies:
- vllm/model_executor/models/
- tests/models/language/pooling/test_embedding.py
@@ -27,10 +27,11 @@ steps:
- pip freeze | grep -E 'torch'
- pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 2
+ mirror:
+ torch_nightly: {}
- label: Language Models Tests (Hybrid) %N
timeout_in_minutes: 75
- torch_nightly: true
source_file_dependencies:
- vllm/
- tests/models/language/generation
@@ -42,6 +43,8 @@ steps:
# Shard hybrid language model tests
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 2
+ mirror:
+ torch_nightly: {}
- label: Language Models Test (Extended Generation) # 80min
timeout_in_minutes: 110
@@ -62,7 +65,7 @@ steps:
- image-build-amd
commands:
- uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
- - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
- label: Language Models Test (PPL)
diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml
index 3bf907bb6c5c..245ef24026d2 100644
--- a/.buildkite/test_areas/models_multimodal.yaml
+++ b/.buildkite/test_areas/models_multimodal.yaml
@@ -28,6 +28,7 @@ steps:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
+ - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
mirror:
amd:
device: mi325_1
@@ -56,7 +57,8 @@ steps:
- tests/models/multimodal
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
+ - pytest models/multimodal/generation/test_memory_leak.py -m core_model
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
mirror:
amd:
diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml
index ad538e919185..a3648219d89d 100644
--- a/.buildkite/test_areas/pytorch.yaml
+++ b/.buildkite/test_areas/pytorch.yaml
@@ -64,6 +64,6 @@ steps:
device: h200_18gb
soft_fail: true
source_file_dependencies:
- - requirements/nightly_torch_test.txt
+ - requirements/test/nightly-torch.txt
commands:
- bash standalone_tests/pytorch_nightly_dependency.sh
diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml
index 0a395ea55883..a42d59b021c6 100644
--- a/.buildkite/test_areas/quantization.yaml
+++ b/.buildkite/test_areas/quantization.yaml
@@ -9,7 +9,7 @@ steps:
- vllm/model_executor/layers/quantization
- tests/quantization
commands:
- # temporary install here since we need nightly, will move to requirements/test.in
+ # temporary install here since we need nightly, will move to requirements/test/cuda.in
# after torchao 0.12 release, and pin a working version of torchao nightly here
# since torchao nightly is only compatible with torch nightly currently
diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml
index a0b730968675..05925da0da01 100644
--- a/.buildkite/test_areas/spec_decode.yaml
+++ b/.buildkite/test_areas/spec_decode.yaml
@@ -12,6 +12,17 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
+- label: Spec Decode Eagle Nightly B200
+ timeout_in_minutes: 30
+ device: b200
+ optional: true
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
+
- label: Spec Decode Speculators + MTP
timeout_in_minutes: 30
device: h200_18gb
@@ -23,6 +34,18 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
+- label: Spec Decode Speculators + MTP Nightly B200
+ timeout_in_minutes: 30
+ device: b200
+ optional: true
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
+
- label: Spec Decode Ngram + Suffix
timeout_in_minutes: 30
device: h200_18gb
@@ -42,3 +65,27 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
+
+- label: Spec Decode Draft Model Nightly B200
+ timeout_in_minutes: 30
+ device: b200
+ optional: true
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
+
+- label: DFlash Speculators Correctness
+ timeout_in_minutes: 30
+ device: h100
+ optional: true
+ num_devices: 1
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/model_executor/models/qwen3_dflash.py
+ - tests/v1/spec_decode/test_speculators_dflash.py
+ commands:
+ - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
+ - pytest -v -s v1/spec_decode/test_speculators_dflash.py -m slow_test
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index a9c1c17128b8..a20c5e7e9dce 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -3,7 +3,7 @@
# This lists cover the "core" components of vLLM that require careful review
/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng @vadiklyutiy
-/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery
+/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
/vllm/lora @jeejeelee
/vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
/vllm/model_executor/layers/fused_moe @mgoin @pavanimajety
@@ -44,8 +44,9 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
/vllm/pooling_params.py @noooop @DarkLight1337
/vllm/tokenizers @DarkLight1337 @njhill
/vllm/renderers @DarkLight1337 @njhill
-/vllm/reasoning @aarnphm @chaunceyjiang
-/vllm/tool_parsers @aarnphm @chaunceyjiang
+/vllm/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
+/vllm/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
+/vllm/parser @aarnphm @chaunceyjiang @sfeng33 @bbrowning
# vLLM V1
/vllm/v1/attention @LucasWilkinson @MatthewBonanni
@@ -91,7 +92,10 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
/tests/v1/kv_connector/nixl_integration @NickLucche
/tests/v1/kv_connector @ApostaC @orozery
/tests/v1/kv_offload @ApostaC @orozery
-/tests/v1/determinism @yewentao256
+/tests/v1/determinism @yewentao256
+/tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
+/tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
+/tests/tool_use @aarnphm @chaunceyjiang @sfeng33 @bbrowning
# Transformers modeling backend
/vllm/model_executor/models/transformers @hmellor
@@ -120,16 +124,16 @@ mkdocs.yaml @hmellor
/tools/pre_commit @hmellor
# CPU
-/vllm/v1/worker/cpu* @bigPYJ1151
+/vllm/v1/worker/cpu* @bigPYJ1151 @xuechendi
/csrc/cpu @bigPYJ1151
-/vllm/platforms/cpu.py @bigPYJ1151
+/vllm/platforms/cpu.py @bigPYJ1151 @xuechendi
/cmake/cpu_extension.cmake @bigPYJ1151
-/docker/Dockerfile.cpu @bigPYJ1151
+/docker/Dockerfile.cpu @bigPYJ1151 @xuechendi
# Intel GPU
-/vllm/v1/worker/xpu* @jikunshang
-/vllm/platforms/xpu.py @jikunshang
-/docker/Dockerfile.xpu @jikunshang
+/vllm/v1/worker/xpu* @jikunshang @xuechendi
+/vllm/platforms/xpu.py @jikunshang @xuechendi
+/docker/Dockerfile.xpu @jikunshang @xuechendi
# Nemotron-specific files
/vllm/model_executor/models/*nemotron* @tomeras91
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 8043df65d558..8a3934670e44 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -15,7 +15,6 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
- [ ] The test plan, such as providing test command.
- [ ] The test results, such as pasting the results comparison before and after, or e2e results
- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model.
-- [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft in the [Google Doc](https://docs.google.com/document/d/1YyVqrgX4gHTtrstbq8oWUImOyPCKSGnJ7xtTpmXzlRs/edit?tab=t.0).
**BEFORE SUBMITTING, PLEASE READ ** (anything written below this line will be removed by GitHub Actions)
diff --git a/.github/mergify.yml b/.github/mergify.yml
index e8ef4d49dd00..b96d6b81ac07 100644
--- a/.github/mergify.yml
+++ b/.github/mergify.yml
@@ -18,7 +18,7 @@ pull_request_rules:
- name: comment-pre-commit-failure
description: Comment on PR when pre-commit check fails
conditions:
- - status-failure=pre-commit
+ - check-failure=pre-commit
- -closed
- -draft
actions:
@@ -51,7 +51,7 @@ pull_request_rules:
- name: comment-dco-failure
description: Comment on PR when DCO check fails
conditions:
- - status-failure=dco
+ - check-failure=dco
- -closed
- -draft
actions:
@@ -83,8 +83,8 @@ pull_request_rules:
- or:
- files~=^examples/.*deepseek.*\.py
- files~=^tests/.*deepseek.*\.py
- - files~=^vllm/entrypoints/openai/tool_parsers/.*deepseek.*\.py
- files~=^vllm/model_executor/models/.*deepseek.*\.py
+ - files~=^vllm/tool_parsers/.*deepseek.*\.py
- files~=^vllm/reasoning/.*deepseek.*\.py
- files~=^vllm/transformers_utils/.*deepseek.*\.py
- title~=(?i)DeepSeek
@@ -110,9 +110,10 @@ pull_request_rules:
- or:
- files~=^examples/.*llama.*\.py
- files~=^tests/.*llama.*\.py
- - files~=^vllm/entrypoints/openai/tool_parsers/llama.*\.py
- files~=^vllm/model_executor/models/.*llama.*\.py
- - files~=^vllm/transformers_utils/configs/.*llama.*\.py
+ - files~=^vllm/reasoning/.*llama.*\.py
+ - files~=^vllm/tool_parsers/.*llama.*\.py
+ - files~=^vllm/transformers_utils/.*llama.*\.py
- title~=(?i)llama
actions:
label:
@@ -133,6 +134,23 @@ pull_request_rules:
add:
- multi-modality
+- name: label-mistral
+ description: Automatically apply mistral label
+ conditions:
+ - label != stale
+ - or:
+ - files~=^examples/.*mistral.*\.py
+ - files~=^tests/.*mistral.*\.py
+ - files~=^vllm/model_executor/models/.*mistral.*\.py
+ - files~=^vllm/reasoning/.*mistral.*\.py
+ - files~=^vllm/tool_parsers/.*mistral.*\.py
+ - files~=^vllm/transformers_utils/.*mistral.*\.py
+ - title~=(?i)Mistral
+ actions:
+ label:
+ add:
+ - mistral
+
- name: label-new-model
description: Automatically apply new-model label
conditions:
@@ -167,7 +185,9 @@ pull_request_rules:
- files~=^examples/.*qwen.*\.py
- files~=^tests/.*qwen.*\.py
- files~=^vllm/model_executor/models/.*qwen.*\.py
+ - files~=^vllm/tool_parsers/.*qwen.*\.py
- files~=^vllm/reasoning/.*qwen.*\.py
+ - files~=^vllm/transformers_utils/.*qwen.*\.py
- title~=(?i)Qwen
actions:
label:
@@ -242,8 +262,9 @@ pull_request_rules:
- files~=^docker/Dockerfile.xpu
- files~=^\\.buildkite/intel_jobs/
- files=\.buildkite/ci_config_intel.yaml
- - files=vllm/model_executor/layers/fused_moe/xpu_fused_moe.py
+ - files=vllm/model_executor/layers/fused_moe/experts/xpu_moe.py
- files=vllm/model_executor/kernels/linear/mixed_precision/xpu.py
+ - files=vllm/model_executor/kernels/linear/mxfp8/xpu.py
- files=vllm/model_executor/kernels/linear/scaled_mm/xpu.py
- files=vllm/distributed/device_communicators/xpu_communicator.py
- files=vllm/v1/attention/backends/mla/xpu_mla_sparse.py
@@ -251,6 +272,7 @@ pull_request_rules:
- files=vllm/v1/worker/xpu_worker.py
- files=vllm/v1/worker/xpu_model_runner.py
- files=vllm/_xpu_ops.py
+ - files=vllm/kernels/xpu_ops.py
- files~=^vllm/lora/ops/xpu_ops
- files=vllm/lora/punica_wrapper/punica_xpu.py
- files=vllm/platforms/xpu.py
@@ -258,7 +280,6 @@ pull_request_rules:
- title~=(?i)XPU
- title~=(?i)Intel
- title~=(?i)BMG
- - title~=(?i)Arc
actions:
label:
add:
@@ -378,17 +399,18 @@ pull_request_rules:
add:
- tool-calling
-- name: auto-rebase if approved, ready, and 40 commits behind main
+- name: auto-rebase to keep merge candidate within 1 day behind main
conditions:
- base = main
- label=ready
- "#approved-reviews-by >= 1"
- - "#commits-behind >= 40"
+ - "#commits-behind >= 50"
+ - "#check-failure = 0"
- -closed
- -draft
- -conflict
actions:
- rebase: {}
+ update: {}
- name: ping author on conflicts and add 'needs-rebase' label
conditions:
diff --git a/.github/workflows/issue_autolabel.yml b/.github/workflows/issue_autolabel.yml
index 2cb5c176ae0a..3efa582f6702 100644
--- a/.github/workflows/issue_autolabel.yml
+++ b/.github/workflows/issue_autolabel.yml
@@ -320,20 +320,25 @@ jobs:
script: |
// Configuration: Map labels to GitHub users to CC
// You can add multiple users per label, and multiple label configurations
+ // {users} will be replaced with @mentions
const ccConfig = {
rocm: {
- users: ['hongxiayang', 'tjtanaa', 'vllmellm'], // Add more users as needed: ['user1', 'user2', 'user3']
- message: 'CC {users} for ROCm-related issue' // {users} will be replaced with @mentions
+ users: ['hongxiayang', 'tjtanaa', 'vllmellm'],
+ message: 'CC {users} for ROCm-related issue',
+ },
+ mistral: {
+ users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
+ message: 'CC {users} for Mistral-related issue',
},
// Add more label -> user mappings here
// Example:
// cuda: {
// users: ['user1', 'user2'],
- // message: 'CC {users} for CUDA-related issue'
+ // message: 'CC {users} for CUDA-related issue',
// },
// performance: {
// users: ['perfexpert'],
- // message: 'CC {users} for performance issue'
+ // message: 'CC {users} for performance issue',
// },
};
diff --git a/.github/workflows/macos-smoke-test.yml b/.github/workflows/macos-smoke-test.yml
index 3c1a50bf8085..ea1c8b0feac3 100644
--- a/.github/workflows/macos-smoke-test.yml
+++ b/.github/workflows/macos-smoke-test.yml
@@ -32,7 +32,7 @@ jobs:
- name: Install dependencies and build vLLM
run: |
- uv pip install -r requirements/cpu-build.txt --index-strategy unsafe-best-match
+ uv pip install -r requirements/build/cpu.txt --index-strategy unsafe-best-match
uv pip install -r requirements/cpu.txt --index-strategy unsafe-best-match
uv pip install -e . --no-build-isolation
env:
@@ -45,6 +45,7 @@ jobs:
- name: Smoke test vllm serve
run: |
# Start server in background
+ VLLM_CPU_KVCACHE_SPACE=1 \
vllm serve Qwen/Qwen3-0.6B \
--max-model-len=2K \
--load-format=dummy \
diff --git a/.github/workflows/new_pr_bot.yml b/.github/workflows/new_pr_bot.yml
index ef5e30952c62..27100f9f4da0 100644
--- a/.github/workflows/new_pr_bot.yml
+++ b/.github/workflows/new_pr_bot.yml
@@ -62,14 +62,14 @@ jobs:
const prAuthor = context.payload.pull_request.user.login;
const { data: searchResults } = await github.rest.search.issuesAndPullRequests({
- q: `repo:${owner}/${repo} type:pr author:${prAuthor}`,
+ q: `repo:${owner}/${repo} type:pr is:merged author:${prAuthor}`,
per_page: 1,
});
- const authorPRCount = searchResults.total_count;
- console.log(`Found ${authorPRCount} PRs by ${prAuthor}`);
+ const mergedPRCount = searchResults.total_count;
+ console.log(`Found ${mergedPRCount} merged PRs by ${prAuthor}`);
- if (authorPRCount === 1) {
+ if (mergedPRCount === 0) {
console.log(`Posting welcome comment for first-time contributor: ${prAuthor}`);
await github.rest.issues.createComment({
owner,
@@ -98,5 +98,5 @@ jobs:
].join('\n'),
});
} else {
- console.log(`Skipping comment for ${prAuthor} - not their first PR (${authorPRCount} PRs found)`);
+ console.log(`Skipping comment for ${prAuthor} - not a first-time contributor (${mergedPRCount} merged PRs)`);
}
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index bbc55e9532b9..8ab8d3e7035f 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -2,6 +2,7 @@ name: pre-commit
on:
pull_request:
+ types: [opened, synchronize, reopened, labeled]
push:
branches: [main]
@@ -15,7 +16,11 @@ permissions:
jobs:
pre-run-check:
- if: github.event_name == 'pull_request'
+ if: >-
+ github.event_name == 'pull_request' &&
+ (github.event.action != 'labeled' ||
+ github.event.label.name == 'ready' ||
+ github.event.label.name == 'verified')
runs-on: ubuntu-latest
steps:
- name: Check PR label and author merge count
@@ -44,7 +49,12 @@ jobs:
pre-commit:
needs: pre-run-check
- if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
+ if: >-
+ always() &&
+ (github.event.action != 'labeled' ||
+ github.event.label.name == 'ready' ||
+ github.event.label.name == 'verified') &&
+ (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build.sh
index c69ebbb42da5..5bb9f7196806 100644
--- a/.github/workflows/scripts/build.sh
+++ b/.github/workflows/scripts/build.sh
@@ -9,12 +9,12 @@ PATH=${cuda_home}/bin:$PATH
LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
# Install requirements
-$python_executable -m pip install -r requirements/build.txt -r requirements/cuda.txt
+$python_executable -m pip install -r requirements/build/cuda.txt -r requirements/cuda.txt
# Limit the number of parallel jobs to avoid OOM
export MAX_JOBS=1
# Make sure release wheels are built for the following architectures
-export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
+export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
bash tools/check_repo.sh
diff --git a/.gitignore b/.gitignore
index 7b822165d3eb..134bbc5cc893 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,9 @@ vllm/third_party/triton_kernels/*
# FlashMLA interface copied from source
vllm/third_party/flashmla/flash_mla_interface.py
+# DeepGEMM vendored package built from source
+vllm/third_party/deep_gemm/
+
# triton jit
.triton
@@ -26,6 +29,7 @@ __pycache__/
# Distribution / packaging
.Python
build/
+!requirements/build/
cmake-build-*/
CMakeUserPresets.json
develop-eggs/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f55df24bc6f3..33b1db69dec4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,15 +39,24 @@ repos:
rev: 0.11.1
hooks:
- id: pip-compile
- args: [requirements/test.in, -c, requirements/common.txt, -o, requirements/test.txt, --index-strategy, unsafe-best-match, --torch-backend, cu130, --python-platform, x86_64-manylinux_2_28, --python-version, "3.12"]
- files: ^requirements/test\.(in|txt)$
+ args: [
+ requirements/test/cuda.in,
+ -c, requirements/cuda.txt,
+ -o, requirements/test/cuda.txt,
+ --index-strategy, unsafe-best-match,
+ --torch-backend, cu130,
+ --python-platform, x86_64-manylinux_2_28,
+ --python-version, "3.12",
+ ]
+ files: ^requirements/(common|cuda|test/cuda)\.(in|txt)$
- id: pip-compile
alias: pip-compile-rocm
name: pip-compile-rocm
args: [
- requirements/rocm-test.in, -o, requirements/rocm-test.txt,
- --index-strategy, unsafe-best-match,
+ requirements/test/rocm.in,
-c, requirements/rocm.txt,
+ -o, requirements/test/rocm.txt,
+ --index-strategy, unsafe-best-match,
--python-platform, x86_64-manylinux_2_28,
--python-version, "3.12",
# Exclude torch and CUDA/NVIDIA packages
@@ -59,30 +68,76 @@ repos:
--no-emit-package, cuda-pathfinder,
--no-emit-package, cuda-toolkit,
--no-emit-package, cupy-cuda12x,
+ # nvidia packages (unsuffixed / unified naming)
--no-emit-package, nvidia-cublas,
--no-emit-package, nvidia-cuda-cupti,
--no-emit-package, nvidia-cuda-nvrtc,
--no-emit-package, nvidia-cuda-runtime,
- --no-emit-package, nvidia-cudnn-cu13,
+ --no-emit-package, nvidia-cudnn,
--no-emit-package, nvidia-cufft,
--no-emit-package, nvidia-cufile,
--no-emit-package, nvidia-curand,
--no-emit-package, nvidia-cusolver,
--no-emit-package, nvidia-cusparse,
+ --no-emit-package, nvidia-cusparselt,
+ --no-emit-package, nvidia-nccl,
+ --no-emit-package, nvidia-nvjitlink,
+ --no-emit-package, nvidia-nvshmem,
+ --no-emit-package, nvidia-nvtx,
+ # nvidia cu12 packages
+ --no-emit-package, nvidia-cublas-cu12,
+ --no-emit-package, nvidia-cuda-cupti-cu12,
+ --no-emit-package, nvidia-cuda-nvrtc-cu12,
+ --no-emit-package, nvidia-cuda-runtime-cu12,
+ --no-emit-package, nvidia-cudnn-cu12,
+ --no-emit-package, nvidia-cufft-cu12,
+ --no-emit-package, nvidia-cufile-cu12,
+ --no-emit-package, nvidia-curand-cu12,
+ --no-emit-package, nvidia-cusolver-cu12,
+ --no-emit-package, nvidia-cusparse-cu12,
+ --no-emit-package, nvidia-cusparselt-cu12,
+ --no-emit-package, nvidia-nccl-cu12,
+ --no-emit-package, nvidia-nvjitlink-cu12,
+ --no-emit-package, nvidia-nvshmem-cu12,
+ --no-emit-package, nvidia-nvtx-cu12,
+ # nvidia cu13 packages
+ --no-emit-package, nvidia-cublas-cu13,
+ --no-emit-package, nvidia-cuda-cupti-cu13,
+ --no-emit-package, nvidia-cuda-nvrtc-cu13,
+ --no-emit-package, nvidia-cuda-runtime-cu13,
+ --no-emit-package, nvidia-cudnn-cu13,
+ --no-emit-package, nvidia-cufft-cu13,
+ --no-emit-package, nvidia-cufile-cu13,
+ --no-emit-package, nvidia-curand-cu13,
+ --no-emit-package, nvidia-cusolver-cu13,
+ --no-emit-package, nvidia-cusparse-cu13,
--no-emit-package, nvidia-cusparselt-cu13,
--no-emit-package, nvidia-nccl-cu13,
- --no-emit-package, nvidia-nvjitlink,
+ --no-emit-package, nvidia-nvjitlink-cu13,
--no-emit-package, nvidia-nvshmem-cu13,
- --no-emit-package, nvidia-nvtx,
+ --no-emit-package, nvidia-nvtx-cu13,
+ ]
+ files: ^requirements/(common|rocm|test/rocm)\.(in|txt)$
+ - id: pip-compile
+ alias: pip-compile-xpu
+ name: pip-compile-xpu
+ args: [
+ requirements/test/xpu.in,
+ -c, requirements/xpu.txt,
+ -o, requirements/test/xpu.txt,
+ --index-strategy, unsafe-best-match,
+ --torch-backend, xpu,
+ --python-platform, x86_64-manylinux_2_39,
+ --python-version, "3.12",
]
- files: ^requirements/rocm-test\.(in|txt)$
+ files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
- repo: local
hooks:
- id: format-torch-nightly-test
- name: reformat nightly_torch_test.txt to be in sync with test.in
+ name: reformat test/nightly-torch.txt to be in sync with test/cuda.in
language: python
entry: python tools/pre_commit/generate_nightly_torch_test.py
- files: ^requirements/test\.(in|txt)$
+ files: ^requirements/test/cuda\.(in|txt)$
- id: mypy-local
name: Run mypy locally for lowest supported Python version
entry: python tools/pre_commit/mypy.py 0 "3.10"
diff --git a/AGENTS.md b/AGENTS.md
index 61312b29ef7d..215e4195eb43 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -72,11 +72,11 @@ uv pip install -e . --torch-backend=auto
```bash
# Install test dependencies.
-# requirements/test.txt is pinned to x86_64; on other platforms, use the
+# requirements/test/cuda.txt is pinned to x86_64; on other platforms, use the
# unpinned source file instead:
-uv pip install -r requirements/test.in # resolves for current platform
+uv pip install -r requirements/test/cuda.in # resolves for current platform
# Or on x86_64:
-uv pip install -r requirements/test.txt
+uv pip install -r requirements/test/cuda.txt
# Run a specific test file (use .venv/bin/python directly;
# `source activate` does not persist in non-interactive shells):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48bd2294cef4..fb8a1d7e1e14 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -34,10 +34,10 @@ install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
# Supported python versions. These versions will be searched in order, the
# first match will be selected. These should be kept in sync with setup.py.
#
-set(PYTHON_SUPPORTED_VERSIONS "3.10" "3.11" "3.12" "3.13")
+set(PYTHON_SUPPORTED_VERSIONS "3.10" "3.11" "3.12" "3.13" "3.14")
# Supported AMD GPU architectures.
-set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201")
+set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201")
# ROCm installation prefix. Default to /opt/rocm but allow override via
# -DROCM_PATH=/your/rocm/path when invoking cmake.
@@ -94,12 +94,15 @@ find_package(Torch REQUIRED)
# This check must happen after find_package(Torch) because that's when CMAKE_CUDA_COMPILER_VERSION gets defined
if(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
- set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0;12.1")
+ # starting from CUDA 12.9 and Blackwell (10.0), we use family-specific targets (10.0f, 12.0f, etc)
+ # to support the whole generation without specifying all sub-architectures
+ # see: https://developer.nvidia.com/blog/nvidia-blackwell-and-nvidia-cuda-12-9-introduce-family-specific-architecture-features/
+ set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0")
elseif(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8)
- set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0;12.1")
+ set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;10.3;12.0;12.1")
else()
- set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0")
+ set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.7;8.9;9.0")
endif()
#
@@ -225,8 +228,8 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
# Certain HIP functions are marked as [[nodiscard]], yet vllm ignores the result which generates
# a lot of warnings that always mask real issues. Suppressing until this is properly addressed.
#
- set(CMAKE_${VLLM_GPU_LANG}_FLAGS "${CMAKE_${VLLM_GPU_LANG}_FLAGS} -Wno-unused-result")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result")
+ set(CMAKE_${VLLM_GPU_LANG}_FLAGS "${CMAKE_${VLLM_GPU_LANG}_FLAGS} -Wno-unused-result -Wno-unused-value")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result -Wno-unused-value")
endif()
#
@@ -299,6 +302,7 @@ set(VLLM_EXT_SRC
"csrc/quantization/w8a8/int8/scaled_quant.cu"
"csrc/quantization/w8a8/fp8/common.cu"
"csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
+ "csrc/quantization/fused_kernels/fused_silu_mul_block_quant.cu"
"csrc/quantization/gguf/gguf_kernel.cu"
"csrc/quantization/activation_kernels.cu"
"csrc/cuda_utils_kernels.cu"
@@ -306,6 +310,10 @@ set(VLLM_EXT_SRC
"csrc/torch_bindings.cpp")
if(VLLM_GPU_LANG STREQUAL "CUDA")
+ list(APPEND VLLM_EXT_SRC
+ "csrc/minimax_reduce_rms_kernel.cu"
+ "csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu")
+
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
# Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
@@ -340,8 +348,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
list(APPEND VLLM_EXT_SRC
"csrc/quantization/awq/gemm_kernels.cu"
- "csrc/cutlass_extensions/common.cpp"
- "csrc/quantization/fused_kernels/fused_silu_mul_block_quant.cu")
+ "csrc/cutlass_extensions/common.cpp")
set_gencode_flags_for_srcs(
SRCS "${VLLM_EXT_SRC}"
@@ -921,6 +928,14 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
SRCS "${SRCS}"
CUDA_ARCHS "${FP4_ARCHS}")
list(APPEND VLLM_STABLE_EXT_SRC "${SRCS}")
+ # nvfp4_kv_cache_kernels uses non-stable torch API and is called directly
+ # from cache_kernels.cu, so it belongs in _C rather than _C_stable.
+ set(NVFP4_KV_SRC "csrc/nvfp4_kv_cache_kernels.cu")
+ set_gencode_flags_for_srcs(
+ SRCS "${NVFP4_KV_SRC}"
+ CUDA_ARCHS "${FP4_ARCHS}")
+ target_sources(_C PRIVATE ${NVFP4_KV_SRC})
+ target_compile_definitions(_C PRIVATE ENABLE_NVFP4_SM120=1)
list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4_SM120=1")
list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM120=1")
message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
@@ -942,11 +957,19 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
"csrc/libtorch_stable/quantization/fp4/activation_nvfp4_quant_fusion_kernels.cu"
"csrc/libtorch_stable/quantization/fp4/nvfp4_experts_quant.cu"
"csrc/libtorch_stable/quantization/fp4/nvfp4_scaled_mm_kernels.cu"
- "csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu")
+ "csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu"
+ "csrc/libtorch_stable/quantization/fp4/mxfp4_experts_quant.cu"
+ "csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu")
set_gencode_flags_for_srcs(
SRCS "${SRCS}"
CUDA_ARCHS "${FP4_ARCHS}")
list(APPEND VLLM_STABLE_EXT_SRC "${SRCS}")
+ set(NVFP4_KV_SRC "csrc/nvfp4_kv_cache_kernels.cu")
+ set_gencode_flags_for_srcs(
+ SRCS "${NVFP4_KV_SRC}"
+ CUDA_ARCHS "${FP4_ARCHS}")
+ target_sources(_C PRIVATE ${NVFP4_KV_SRC})
+ target_compile_definitions(_C PRIVATE ENABLE_NVFP4_SM100=1)
list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4_SM100=1")
list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM100=1")
message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
@@ -1030,7 +1053,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
list(APPEND VLLM_MOE_EXT_SRC
"csrc/moe/moe_wna16.cu"
"csrc/moe/grouped_topk_kernels.cu"
- "csrc/moe/router_gemm.cu")
+ "csrc/moe/router_gemm.cu"
+ "csrc/moe/topk_softplus_sqrt_kernels.cu")
endif()
if(VLLM_GPU_LANG STREQUAL "CUDA")
@@ -1222,6 +1246,7 @@ endif()
# For CUDA we also build and ship some external projects.
if (VLLM_GPU_LANG STREQUAL "CUDA")
+ include(cmake/external_projects/deepgemm.cmake)
include(cmake/external_projects/flashmla.cmake)
include(cmake/external_projects/qutlass.cmake)
diff --git a/README.md b/README.md
index 44418b712a26..d5653c76c8d5 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Easy, fast, and cheap LLM serving for everyone!
| Documentation | Blog | Paper | Twitter/X | User Forum | Developer Slack |
-🔥 We have built a vllm website to help you get started with vllm. Please visit [vllm.ai](https://vllm.ai) to learn more.
+🔥 We have built a vLLM website to help you get started with vLLM. Please visit [vllm.ai](https://vllm.ai) to learn more.
For events, please visit [vllm.ai/events](https://vllm.ai/events) to join us.
---
@@ -23,47 +23,54 @@ For events, please visit [vllm.ai/events](https://vllm.ai/events) to join us.
vLLM is a fast and easy-to-use library for LLM inference and serving.
-Originally developed in the [Sky Computing Lab](https://sky.cs.berkeley.edu) at UC Berkeley, vLLM has evolved into a community-driven project with contributions from both academia and industry.
+Originally developed in the [Sky Computing Lab](https://sky.cs.berkeley.edu) at UC Berkeley, vLLM has grown into one of the most active open-source AI projects built and maintained by a diverse community of many dozens of academic institutions and companies from over 2000 contributors.
vLLM is fast with:
- State-of-the-art serving throughput
- Efficient management of attention key and value memory with [**PagedAttention**](https://blog.vllm.ai/2023/06/20/vllm.html)
-- Continuous batching of incoming requests
-- Fast model execution with CUDA/HIP graph
-- Quantizations: [GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), [AutoRound](https://arxiv.org/abs/2309.05516), INT4, INT8, and FP8
-- Optimized CUDA kernels, including integration with FlashAttention and FlashInfer
-- Speculative decoding
-- Chunked prefill
+- Continuous batching of incoming requests, chunked prefill, prefix caching
+- Fast and flexible model execution with piecewise and full CUDA/HIP graphs
+- Quantization: FP8, MXFP8/MXFP4, NVFP4, INT8, INT4, GPTQ/AWQ, GGUF, compressed-tensors, ModelOpt, TorchAO, and [more](https://docs.vllm.ai/en/latest/features/quantization/index.html)
+- Optimized attention kernels including FlashAttention, FlashInfer, TRTLLM-GEN, FlashMLA, and Triton
+- Optimized GEMM/MoE kernels for various precisions using CUTLASS, TRTLLM-GEN, CuTeDSL
+- Speculative decoding including n-gram, suffix, EAGLE, DFlash
+- Automatic kernel generation and graph-level transformations using torch.compile
+- Disaggregated prefill, decode, and encode
vLLM is flexible and easy to use with:
- Seamless integration with popular Hugging Face models
- High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
-- Tensor, pipeline, data and expert parallelism support for distributed inference
+- Tensor, pipeline, data, expert, and context parallelism for distributed inference
- Streaming outputs
-- OpenAI-compatible API server
-- Support for NVIDIA GPUs, AMD CPUs and GPUs, Intel CPUs and GPUs, PowerPC CPUs, Arm CPUs, and TPU. Additionally, support for diverse hardware plugins such as Intel Gaudi, IBM Spyre and Huawei Ascend.
-- Prefix caching support
-- Multi-LoRA support
+- Generation of structured outputs using xgrammar or guidance
+- Tool calling and reasoning parsers
+- OpenAI-compatible API server, plus Anthropic Messages API and gRPC support
+- Efficient multi-LoRA support for dense and MoE layers
+- Support for NVIDIA GPUs, AMD GPUs, and x86/ARM/PowerPC CPUs. Additionally, diverse hardware plugins such as Google TPUs, Intel Gaudi, IBM Spyre, Huawei Ascend, Rebellions NPU, Apple Silicon, MetaX GPU, and more.
-vLLM seamlessly supports most popular open-source models on HuggingFace, including:
+vLLM seamlessly supports 200+ model architectures on Hugging Face, including:
-- Transformer-like LLMs (e.g., Llama)
-- Mixture-of-Expert LLMs (e.g., Mixtral, Deepseek-V2 and V3)
-- Embedding Models (e.g., E5-Mistral)
-- Multi-modal LLMs (e.g., LLaVA)
+- Decoder-only LLMs (e.g., Llama, Qwen, Gemma)
+- Mixture-of-Expert LLMs (e.g., Mixtral, DeepSeek-V3, Qwen-MoE, GPT-OSS)
+- Hybrid attention and state-space models (e.g., Mamba, Qwen3.5)
+- Multi-modal models (e.g., LLaVA, Qwen-VL, Pixtral)
+- Embedding and retrieval models (e.g., E5-Mistral, GTE, ColBERT)
+- Reward and classification models (e.g., Qwen-Math)
Find the full list of supported models [here](https://docs.vllm.ai/en/latest/models/supported_models.html).
## Getting Started
-Install vLLM with `pip` or [from source](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/index.html#build-wheel-from-source):
+Install vLLM with [`uv`](https://docs.astral.sh/uv/) (recommended) or `pip`:
```bash
-pip install vllm
+uv pip install vllm
```
+Or [build from source](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/index.html#build-wheel-from-source) for development.
+
Visit our [documentation](https://docs.vllm.ai/en/latest/) to learn more.
- [Installation](https://docs.vllm.ai/en/latest/getting_started/installation.html)
diff --git a/tests/entrypoints/pooling/pooling/__init__.py b/benchmarks/__init__.py
similarity index 100%
rename from tests/entrypoints/pooling/pooling/__init__.py
rename to benchmarks/__init__.py
diff --git a/benchmarks/attention_benchmarks/mla_runner.py b/benchmarks/attention_benchmarks/mla_runner.py
index f8bc7b4a10ed..b58ddffcd0b1 100644
--- a/benchmarks/attention_benchmarks/mla_runner.py
+++ b/benchmarks/attention_benchmarks/mla_runner.py
@@ -404,6 +404,7 @@ def _build_attention_metadata(
query_start_loc=q_start_gpu,
query_start_loc_cpu=q_start_cpu,
seq_lens=seq_lens_gpu,
+ seq_lens_cpu_upper_bound=seq_lens_cpu,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
slot_mapping=slot_mapping,
diff --git a/benchmarks/kernels/__init__.py b/benchmarks/kernels/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/benchmarks/kernels/benchmark_block_fp8_gemm.py b/benchmarks/kernels/benchmark_block_fp8_gemm.py
index 8d50c3828206..9eddc907b937 100644
--- a/benchmarks/kernels/benchmark_block_fp8_gemm.py
+++ b/benchmarks/kernels/benchmark_block_fp8_gemm.py
@@ -9,11 +9,12 @@
import torch
from vllm.benchmarks.lib.utils import default_vllm_config
-from vllm.model_executor.layers.quantization.utils.fp8_utils import (
- W8A8BlockFp8LinearOp,
+from vllm.model_executor.kernels.linear import (
+ init_fp8_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
+ create_fp8_quant_key,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
CUTLASS_BLOCK_FP8_SUPPORTED,
@@ -70,11 +71,15 @@ def build_w8a8_block_fp8_runner(M, N, K, block_size, device, use_cutlass):
weight_group_shape = GroupShape(block_n, block_k)
act_quant_group_shape = GroupShape(1, block_k) # Per-token, per-group quantization
- linear_op = W8A8BlockFp8LinearOp(
- weight_group_shape=weight_group_shape,
- act_quant_group_shape=act_quant_group_shape,
- cutlass_block_fp8_supported=use_cutlass,
- use_aiter_and_is_supported=False,
+ linear_op = init_fp8_linear_kernel(
+ weight_quant_key=create_fp8_quant_key(
+ static=True, group_shape=weight_group_shape
+ ),
+ activation_quant_key=create_fp8_quant_key(
+ static=False, group_shape=act_quant_group_shape
+ ),
+ out_dtype=torch.get_default_dtype(),
+ module_name="build_w8a8_block_fp8_runner",
)
def run():
diff --git a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py
index 3f80b024e108..03d7fb386f74 100644
--- a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py
+++ b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py
@@ -16,7 +16,7 @@
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
-from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8
+from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
diff --git a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py
index 2d4afd38c097..7379bf858889 100644
--- a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py
+++ b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py
@@ -22,7 +22,7 @@
fp8_w8a8_moe_quant_config,
nvfp4_moe_quant_config,
)
-from vllm.model_executor.layers.fused_moe.cutlass_moe import (
+from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
diff --git a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py
index dd4060bbdb94..04fc2960d1e4 100644
--- a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py
+++ b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py
@@ -13,7 +13,7 @@
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
-from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8
+from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
fused_topk,
diff --git a/benchmarks/kernels/benchmark_moe_align_block_size.py b/benchmarks/kernels/benchmark_moe_align_block_size.py
index 5f9a131f79b0..a340500379a0 100644
--- a/benchmarks/kernels/benchmark_moe_align_block_size.py
+++ b/benchmarks/kernels/benchmark_moe_align_block_size.py
@@ -9,6 +9,7 @@
moe_align_block_size,
)
from vllm.triton_utils import triton
+from vllm.utils.torch_utils import set_random_seed
def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor:
@@ -44,7 +45,7 @@ def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor:
def benchmark(num_tokens, num_experts, topk, ep_size, provider):
"""Benchmark function for Triton."""
block_size = 256
- torch.cuda.manual_seed_all(0)
+ set_random_seed(0)
topk_ids = get_topk_ids(num_tokens, num_experts, topk)
e_map = None
diff --git a/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py b/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py
index 13b97b7696b3..9fcf278f2ef3 100644
--- a/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py
+++ b/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py
@@ -20,7 +20,7 @@
import numpy as np
import torch
-from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
+from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
persistent_masked_m_silu_mul_quant,
)
from vllm.triton_utils import tl, triton
diff --git a/benchmarks/kernels/deepgemm/benchmark_fp8_block_dense_gemm.py b/benchmarks/kernels/deepgemm/benchmark_fp8_block_dense_gemm.py
index 4384d3e56828..c9aaef284d70 100644
--- a/benchmarks/kernels/deepgemm/benchmark_fp8_block_dense_gemm.py
+++ b/benchmarks/kernels/deepgemm/benchmark_fp8_block_dense_gemm.py
@@ -16,6 +16,7 @@
fp8_gemm_nt,
per_block_cast_to_fp8,
)
+from vllm.utils.torch_utils import set_random_seed
def benchmark_shape(
@@ -235,9 +236,7 @@ def run_benchmarks(verbose: bool = False):
torch.backends.cudnn.allow_tf32 = True
# Set seeds for reproducibility
- torch.manual_seed(42)
- torch.cuda.manual_seed(42)
-
+ set_random_seed(42)
# Define benchmark shapes (m, n, k)
shapes = [
(8, 4096, 7168),
diff --git a/benchmarks/kernels/ir/__init__.py b/benchmarks/kernels/ir/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/benchmarks/kernels/ir/bench_ir_ops.py b/benchmarks/kernels/ir/bench_ir_ops.py
new file mode 100644
index 000000000000..b23c4e8ae327
--- /dev/null
+++ b/benchmarks/kernels/ir/bench_ir_ops.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Generic benchmark harness for vLLM IR ops.
+
+Usage:
+ python benchmarks/kernels/ir/bench_ir_ops.py
+ python benchmarks/kernels/ir/bench_ir_ops.py --ops rms_norm
+ python benchmarks/kernels/ir/bench_ir_ops.py --ops rms_norm,silu_mul
+ python benchmarks/kernels/ir/bench_ir_ops.py --no-cuda-graph
+ python benchmarks/kernels/ir/bench_ir_ops.py --ops rms_norm --save-path ./results/
+"""
+
+import argparse
+import contextlib
+import csv
+import dataclasses
+import datetime
+import math
+import os
+import subprocess
+import sys
+import tempfile
+
+# Ensure repo root is on sys.path so `benchmarks` is importable as a package.
+_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
+if _REPO_ROOT not in sys.path:
+ sys.path.insert(0, _REPO_ROOT)
+
+# Suppress noisy C++ warnings from vllm kernel registration (written to fd 2
+# directly by the dynamic linker, so Python-level sys.stderr redirect won't
+# catch them).
+_saved_fd = os.dup(2)
+try:
+ with open(os.devnull, "w") as _devnull:
+ os.dup2(_devnull.fileno(), 2)
+ import torch
+
+ import vllm.kernels # noqa: E402, F401
+finally:
+ os.dup2(_saved_fd, 2)
+ os.close(_saved_fd)
+
+from tqdm import tqdm # noqa: E402
+
+from benchmarks.kernels.ir.shapes import SHAPE_CONFIGS # noqa: E402 # isort: skip
+from vllm.ir.op import IrOp # noqa: E402
+from vllm.platforms import current_platform # noqa: E402
+from vllm.triton_utils import triton # noqa: E402
+
+
+@dataclasses.dataclass(frozen=True)
+class BenchConfig:
+ use_cuda_graph: bool = True
+ warmup: int = 25
+ rep: int = 100
+
+
+def _pkg_version(name: str) -> str:
+ from importlib.metadata import PackageNotFoundError, version
+
+ with contextlib.suppress(PackageNotFoundError):
+ return version(name)
+ return "not installed"
+
+
+_METADATA_LABELS = {
+ "timestamp": "Timestamp",
+ "git_commit": "Git commit",
+ "vllm": "vLLM",
+ "pytorch": "PyTorch",
+ "cuda_runtime": "CUDA runtime",
+ "triton": "Triton",
+ "cutlass": "CUTLASS",
+ "helion": "Helion",
+ "device": "Device",
+ "bench_mode": "Bench mode",
+ "warmup": "Warmup",
+ "rep": "Repetitions",
+}
+
+
+def collect_env_metadata(cfg: BenchConfig) -> dict[str, str]:
+ from vllm.collect_env import get_env_info
+
+ env = get_env_info()
+
+ git_sha = "unknown"
+ with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError):
+ git_sha = (
+ subprocess.check_output(
+ ["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL
+ )
+ .decode()
+ .strip()
+ )
+
+ device_name = current_platform.get_device_name()
+
+ warmup_note = " ms" if not cfg.use_cuda_graph else " ms (ignored)"
+ rep_note = " replays" if cfg.use_cuda_graph else " ms"
+
+ return {
+ "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+ "git_commit": git_sha,
+ "vllm": str(env.vllm_version),
+ "pytorch": str(env.torch_version),
+ "cuda_runtime": str(env.cuda_runtime_version),
+ "triton": triton.__version__,
+ "cutlass": _pkg_version("nvidia-cutlass-dsl"),
+ "helion": _pkg_version("helion"),
+ "device": device_name,
+ "bench_mode": "cuda_graph" if cfg.use_cuda_graph else "eager",
+ "warmup": f"{cfg.warmup}{warmup_note}",
+ "rep": f"{cfg.rep}{rep_note}",
+ }
+
+
+def print_metadata(metadata: dict[str, str]):
+ print("=" * 60)
+ for key, val in metadata.items():
+ print(f"{_METADATA_LABELS.get(key, key) + ':':<16}{val}")
+ print("=" * 60)
+
+
+def _clone_args(args: tuple) -> tuple:
+ return tuple(a.clone() if isinstance(a, torch.Tensor) else a for a in args)
+
+
+# TODO(gmagogsfm): When the `maybe_inplace` PR lands, ops marked as
+# inplace=True will mutate bench_args across iterations. Both CUDA graph
+# and eager modes will accumulate drift from repeated in-place mutation.
+# We need to re-clone inputs per iteration for inplace ops.
+def _bench_one(fn, args, cfg: BenchConfig) -> float:
+ bench_args = _clone_args(args)
+ bench_fn = lambda: fn(*bench_args)
+
+ if cfg.use_cuda_graph:
+ ms = triton.testing.do_bench_cudagraph(bench_fn, rep=cfg.rep, quantiles=[0.5])
+ else:
+ ms = triton.testing.do_bench(
+ bench_fn, warmup=cfg.warmup, rep=cfg.rep, quantiles=[0.5]
+ )
+ return ms * 1000
+
+
+# TODO(gmagogsfm): Once compiled native implementation lands (#38775),
+# the benchmark baseline should be the compiled native (what vLLM runs by
+# default) rather than the uncompiled native implementation.
+def collect_timings(
+ op: IrOp, shape_configs: list[dict], cfg: BenchConfig
+) -> tuple[list[str], list[str], dict[str, dict[str, float]]]:
+ def fmt(v) -> str:
+ return str(v).split(".")[-1] if isinstance(v, torch.dtype) else str(v)
+
+ case_names = [
+ "_".join(f"{k}={fmt(v)}" for k, v in kwargs.items()) for kwargs in shape_configs
+ ]
+ providers = [n for n, impl in op.impls.items() if impl.supported]
+
+ results: dict[str, dict[str, float]] = {c: {} for c in case_names}
+ for provider in providers:
+ impl = op.impls[provider]
+ desc = f"{op.name} / {provider}"
+ for case_name, kwargs in tqdm(
+ zip(case_names, shape_configs),
+ desc=desc,
+ total=len(case_names),
+ unit=" cases",
+ ):
+ args = op.generate_inputs(**kwargs)
+ if impl.supports_args(*args):
+ results[case_name][provider] = _bench_one(impl.impl_fn, args, cfg)
+ else:
+ results[case_name][provider] = float("nan")
+
+ return case_names, providers, results
+
+
+def analyze_results(
+ op_name: str,
+ case_names: list[str],
+ providers: list[str],
+ results: dict[str, dict[str, float]],
+) -> tuple[list[dict[str, str]], list[dict[str, str]], list[str]]:
+ native_col = "native"
+ non_native = [p for p in providers if p != native_col]
+
+ header_cols = ["case"]
+ for p in providers:
+ header_cols.append(f"{p} (us)")
+ for p in non_native:
+ header_cols.append(f"{p} speedup")
+
+ detail_rows: list[dict[str, str]] = []
+ speedup_data: dict[str, list[tuple[float, str]]] = {p: [] for p in non_native}
+
+ for case_name in case_names:
+ timings = results[case_name]
+ row: dict[str, str] = {"case": case_name}
+
+ for p in providers:
+ val = timings.get(p, float("nan"))
+ row[f"{p} (us)"] = f"{val:.2f}" if not math.isnan(val) else "n/a"
+
+ native_us = timings.get(native_col, float("nan"))
+ for p in non_native:
+ p_us = timings.get(p, float("nan"))
+ if not math.isnan(native_us) and not math.isnan(p_us) and p_us > 0:
+ speedup = native_us / p_us
+ row[f"{p} speedup"] = f"{speedup:.2f}x"
+ speedup_data[p].append((speedup, case_name))
+ else:
+ row[f"{p} speedup"] = "n/a"
+
+ detail_rows.append(row)
+
+ summary_rows: list[dict[str, str]] = []
+ for p in non_native:
+ entries = speedup_data[p]
+ if not entries:
+ continue
+ speedups = [s for s, _ in entries]
+ geomean = math.exp(sum(math.log(s) for s in speedups) / len(speedups))
+ best_val, best_case = max(entries)
+ worst_val, worst_case = min(entries)
+ wins = sum(1 for s in speedups if s > 1.0)
+ losses = sum(1 for s in speedups if s < 1.0)
+ total = len(speedups)
+
+ print(f"\n{p} vs native ({wins}/{total} faster, {losses}/{total} slower):")
+ print(f" geomean speedup: {geomean:.2f}x")
+ print(f" best: {best_val:.2f}x ({best_case})")
+ print(f" worst: {worst_val:.2f}x ({worst_case})")
+
+ summary_rows.append(
+ {
+ "op": op_name,
+ "provider": p,
+ "geomean_speedup": f"{geomean:.2f}",
+ "best_speedup": f"{best_val:.2f}",
+ "best_case": best_case,
+ "worst_speedup": f"{worst_val:.2f}",
+ "worst_case": worst_case,
+ "wins": str(wins),
+ "losses": str(losses),
+ "total": str(total),
+ }
+ )
+
+ return detail_rows, summary_rows, header_cols
+
+
+def write_csv(path: str, rows: list[dict[str, str]], fieldnames: list[str]):
+ with open(path, "w", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def save_results(
+ save_dir: str,
+ op_name: str,
+ detail_rows: list[dict[str, str]],
+ header_cols: list[str],
+ all_summary_rows: list[dict[str, str]],
+ metadata: dict[str, str],
+):
+ write_csv(
+ os.path.join(save_dir, f"{op_name}_detail.csv"),
+ detail_rows,
+ header_cols,
+ )
+ if all_summary_rows:
+ write_csv(
+ os.path.join(save_dir, "summary.csv"),
+ all_summary_rows,
+ list(all_summary_rows[0].keys()),
+ )
+ write_csv(
+ os.path.join(save_dir, "metadata.csv"),
+ [metadata],
+ list(metadata.keys()),
+ )
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Benchmark vLLM IR ops")
+ parser.add_argument(
+ "--ops",
+ type=str,
+ default=None,
+ help="Comma-separated list of op names to benchmark (substring match)",
+ )
+ parser.add_argument(
+ "--no-cuda-graph",
+ action="store_true",
+ help="Disable CUDA graph; use do_bench with L2 cache flushing instead",
+ )
+ parser.add_argument(
+ "--warmup",
+ type=int,
+ default=25,
+ help="Warmup time in ms (do_bench) or ignored with CUDA graph (default: 25)",
+ )
+ parser.add_argument(
+ "--rep",
+ type=int,
+ default=100,
+ help="Repetition time in ms (do_bench) or number of graph replays "
+ "(do_bench_cudagraph) (default: 100)",
+ )
+ parser.add_argument(
+ "--save-path",
+ type=str,
+ default=None,
+ help="Directory to save results (default: auto-created temp dir)",
+ )
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+ cfg = BenchConfig(
+ use_cuda_graph=not args.no_cuda_graph,
+ warmup=args.warmup,
+ rep=args.rep,
+ )
+
+ torch.set_default_device(current_platform.device_type)
+
+ metadata = collect_env_metadata(cfg)
+ print_metadata(metadata)
+
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+ save_dir = args.save_path or os.path.join(
+ tempfile.gettempdir(), f"vllm_ir_bench_{timestamp}"
+ )
+ os.makedirs(save_dir, exist_ok=True)
+
+ op_filters = [f.strip() for f in args.ops.split(",")] if args.ops else None
+ all_summary_rows: list[dict[str, str]] = []
+
+ for op in IrOp.registry.values():
+ if op_filters and not any(f in op.name for f in op_filters):
+ continue
+ if not op.has_input_generator:
+ print(f"Skipping op '{op.name}': no input generator registered")
+ continue
+ if op.name not in SHAPE_CONFIGS:
+ raise RuntimeError(
+ f"No benchmark shape config for op '{op.name}'. "
+ f"Add it to benchmarks/kernels/ir/shapes.py"
+ )
+
+ case_names, providers, results = collect_timings(
+ op, SHAPE_CONFIGS[op.name], cfg
+ )
+ detail_rows, summary_rows, header_cols = analyze_results(
+ op.name, case_names, providers, results
+ )
+ all_summary_rows.extend(summary_rows)
+
+ save_results(
+ save_dir,
+ op.name,
+ detail_rows,
+ header_cols,
+ all_summary_rows,
+ metadata,
+ )
+
+ print(f"\nResults saved to: {save_dir}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/benchmarks/kernels/ir/shapes.py b/benchmarks/kernels/ir/shapes.py
new file mode 100644
index 000000000000..6cc44cf6cec1
--- /dev/null
+++ b/benchmarks/kernels/ir/shapes.py
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Shape configurations for IR op benchmarks.
+"""
+
+import torch
+
+NUM_TOKENS = [1, 2, 4, 16, 64, 256, 1024, 4096, 16384]
+COMMON_HIDDEN_SIZES = [
+ 2048, # Llama 3.2 1B, Qwen 3 MoE 30B-A3B, Gemma 3n
+ 3072, # Gemma 7B/9B
+ 4096, # Llama 3 8B, Qwen 3 8B, Mistral 7B
+ 5120, # Llama 4 Scout 17B-16E
+ 7168, # DeepSeek V3
+ 8192, # Llama 3 70B
+ 16384, # Llama 3 405B
+]
+
+# Each entry maps an op name to a list of kwarg dicts that will be passed
+# to that op's registered input generator via op.generate_inputs(**kwargs).
+SHAPE_CONFIGS: dict[str, list[dict]] = {
+ "rms_norm": [
+ {"num_tokens": n, "hidden_size": d, "dtype": dtype}
+ for dtype in [torch.float16, torch.bfloat16, torch.float32]
+ for d in COMMON_HIDDEN_SIZES
+ for n in NUM_TOKENS
+ ],
+}
diff --git a/benchmarks/multi_turn/benchmark_serving_multi_turn.py b/benchmarks/multi_turn/benchmark_serving_multi_turn.py
index e23f6b923f1b..881039f43f07 100644
--- a/benchmarks/multi_turn/benchmark_serving_multi_turn.py
+++ b/benchmarks/multi_turn/benchmark_serving_multi_turn.py
@@ -1439,6 +1439,12 @@ async def main() -> None:
action="store_true",
help="Export summary to Excel file (optional)",
)
+ parser.add_argument(
+ "--stats-json-output",
+ type=str,
+ default=None,
+ help="Export per-request stats (ttft_ms, tpot_ms, etc.) to a JSON file",
+ )
parser.add_argument(
"-v",
"--verbose",
@@ -1651,6 +1657,19 @@ async def main() -> None:
warmup_runtime_sec=warmup_runtime_sec,
)
+ if args.stats_json_output is not None:
+ # Export per-request metrics as a JSON array for downstream analysis.
+ stats_data = [s._asdict() for s in client_metrics]
+ logger.info(
+ f"{Color.GREEN}Writing per-request stats JSON: "
+ f"{args.stats_json_output}{Color.RESET}"
+ )
+ os.makedirs(
+ os.path.dirname(os.path.abspath(args.stats_json_output)), exist_ok=True
+ )
+ with open(args.stats_json_output, "w") as f:
+ json.dump(stats_data, f, indent=2)
+
if args.output_file is not None:
# Write a JSON file with the updated conversations
# The "assistant" content will contain the answers from the tested LLM
diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake
index 1b3f0d5ad833..8535186cc1ec 100644
--- a/cmake/cpu_extension.cmake
+++ b/cmake/cpu_extension.cmake
@@ -30,6 +30,21 @@ else()
list(APPEND CXX_COMPILE_FLAGS
"-fopenmp"
"-DVLLM_CPU_EXTENSION")
+
+ # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0)
+ # and create a local shim dir with it
+ vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR)
+
+ find_library(OPEN_MP
+ NAMES gomp
+ PATHS ${VLLM_TORCH_GOMP_SHIM_DIR}
+ NO_DEFAULT_PATH
+ REQUIRED
+ )
+ # Set LD_LIBRARY_PATH to include the shim dir at build time to use the same libgomp as PyTorch
+ if (OPEN_MP)
+ set(ENV{LD_LIBRARY_PATH} "${VLLM_TORCH_GOMP_SHIM_DIR}:$ENV{LD_LIBRARY_PATH}")
+ endif()
endif()
if (NOT MACOSX_FOUND)
@@ -146,16 +161,49 @@ elseif (S390_FOUND)
"-mtune=native")
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
message(STATUS "RISC-V detected")
- if(RVV_BF16_FOUND)
- message(STATUS "BF16 extension detected")
- set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl128b -mrvv-vector-bits=zvl -mabi=lp64d)
- add_compile_definitions(RISCV_BF16_SUPPORT)
- elseif (RVV_FP16_FOUND)
- message(WARNING "BF16 functionality is not available")
- set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl128b -mrvv-vector-bits=zvl -mabi=lp64d)
+ # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo
+ # by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256.
+ if(NOT DEFINED VLLM_RVV_VLEN)
+ # Auto-detect: find the largest zvlb in /proc/cpuinfo isa line.
+ if(EXISTS /proc/cpuinfo)
+ file(READ /proc/cpuinfo _cpuinfo)
+ set(_best 0)
+ foreach(_n IN ITEMS 128 256 512 1024)
+ if(_cpuinfo MATCHES "zvl${_n}b")
+ set(_best ${_n})
+ endif()
+ endforeach()
+ if(_best GREATER 0)
+ set(VLLM_RVV_VLEN ${_best})
+ endif()
+ endif()
+ # If auto-detect failed (no /proc/cpuinfo or no zvlb reported)
+ # but the compiler supports RVV, require explicit specification.
+ if(NOT DEFINED VLLM_RVV_VLEN AND (RVV_FP16_FOUND OR RVV_BF16_FOUND))
+ message(FATAL_ERROR
+ "RISC-V RVV is available but VLEN could not be auto-detected. "
+ "Please specify VLEN explicitly:\n"
+ " -DVLLM_RVV_VLEN=128 (for VLEN=128 hardware)\n"
+ " -DVLLM_RVV_VLEN=256 (for VLEN=256 hardware, e.g. Spacemit X100)\n"
+ " -DVLLM_RVV_VLEN=0 (force scalar, no RVV)")
+ endif()
+ endif()
+ if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0)
+ message(STATUS "RISC-V RVV VLEN=${VLLM_RVV_VLEN}")
+ if(RVV_BF16_FOUND)
+ message(STATUS "BF16 extension detected")
+ set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d)
+ add_compile_definitions(RISCV_BF16_SUPPORT)
+ elseif(RVV_FP16_FOUND)
+ message(WARNING "BF16 functionality is not available")
+ set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d)
+ else()
+ message(STATUS "compile riscv with scalar (no FP16/BF16)")
+ set(MARCH_FLAGS -march=rv64gc)
+ endif()
else()
message(STATUS "compile riscv with scalar")
- list(APPEND CXX_COMPILE_FLAGS "-march=rv64gc")
+ set(MARCH_FLAGS -march=rv64gc)
endif()
list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS})
else()
@@ -175,20 +223,6 @@ if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND
if(NOT NPROC)
set(NPROC 4)
endif()
- # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0)
- # and create a local shim dir with it
- vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR)
-
- find_library(OPEN_MP
- NAMES gomp
- PATHS ${VLLM_TORCH_GOMP_SHIM_DIR}
- NO_DEFAULT_PATH
- REQUIRED
- )
- # Set LD_LIBRARY_PATH to include the shim dir at build time to use the same libgomp as PyTorch
- if (OPEN_MP)
- set(ENV{LD_LIBRARY_PATH} "${VLLM_TORCH_GOMP_SHIM_DIR}:$ENV{LD_LIBRARY_PATH}")
- endif()
# Fetch and populate ACL
if(DEFINED ENV{ACL_ROOT_DIR} AND IS_DIRECTORY "$ENV{ACL_ROOT_DIR}")
@@ -349,6 +383,7 @@ endif()
set(VLLM_EXT_SRC
"csrc/cpu/activation.cpp"
"csrc/cpu/utils.cpp"
+ "csrc/cpu/spec_decode_utils.cpp"
"csrc/cpu/layernorm.cpp"
"csrc/cpu/mla_decode.cpp"
"csrc/cpu/pos_encoding.cpp"
@@ -359,6 +394,7 @@ set(VLLM_EXT_SRC
if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
set(VLLM_EXT_SRC
"csrc/cpu/shm.cpp"
+ "csrc/cpu/activation_lut_bf16.cpp"
${VLLM_EXT_SRC})
endif()
@@ -383,6 +419,7 @@ if (ENABLE_X86_ISA)
"csrc/cpu/cpu_wna16.cpp"
"csrc/cpu/cpu_fused_moe.cpp"
"csrc/cpu/utils.cpp"
+ "csrc/cpu/spec_decode_utils.cpp"
"csrc/cpu/cpu_attn.cpp"
"csrc/cpu/dnnl_kernels.cpp"
"csrc/cpu/torch_bindings.cpp"
@@ -395,6 +432,7 @@ if (ENABLE_X86_ISA)
set(VLLM_EXT_SRC_AVX2
"csrc/cpu/utils.cpp"
+ "csrc/cpu/spec_decode_utils.cpp"
"csrc/cpu/cpu_attn.cpp"
"csrc/cpu/torch_bindings.cpp"
# TODO: Remove these files
diff --git a/cmake/external_projects/deepgemm.cmake b/cmake/external_projects/deepgemm.cmake
new file mode 100644
index 000000000000..0d7ea43fb7d0
--- /dev/null
+++ b/cmake/external_projects/deepgemm.cmake
@@ -0,0 +1,156 @@
+include(FetchContent)
+
+# If DEEPGEMM_SRC_DIR is set, DeepGEMM is built from that directory
+# instead of downloading.
+# It can be set as an environment variable or passed as a cmake argument.
+# The environment variable takes precedence.
+if (DEFINED ENV{DEEPGEMM_SRC_DIR})
+ set(DEEPGEMM_SRC_DIR $ENV{DEEPGEMM_SRC_DIR})
+endif()
+
+if(DEEPGEMM_SRC_DIR)
+ FetchContent_Declare(
+ deepgemm
+ SOURCE_DIR ${DEEPGEMM_SRC_DIR}
+ CONFIGURE_COMMAND ""
+ BUILD_COMMAND ""
+ )
+else()
+ # This ref should be kept in sync with tools/install_deepgemm.sh
+ FetchContent_Declare(
+ deepgemm
+ GIT_REPOSITORY https://github.com/deepseek-ai/DeepGEMM.git
+ GIT_TAG 891d57b4db1071624b5c8fa0d1e51cb317fa709f
+ GIT_SUBMODULES "third-party/cutlass" "third-party/fmt"
+ GIT_PROGRESS TRUE
+ CONFIGURE_COMMAND ""
+ BUILD_COMMAND ""
+ )
+endif()
+
+# Use FetchContent_Populate (not MakeAvailable) to avoid processing
+# DeepGEMM's own CMakeLists.txt which has incompatible find_package calls.
+FetchContent_GetProperties(deepgemm)
+if(NOT deepgemm_POPULATED)
+ FetchContent_Populate(deepgemm)
+endif()
+message(STATUS "DeepGEMM is available at ${deepgemm_SOURCE_DIR}")
+
+# DeepGEMM requires CUDA 12.3+ for SM90, 12.9+ for SM100
+set(DEEPGEMM_SUPPORT_ARCHS)
+if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3)
+ list(APPEND DEEPGEMM_SUPPORT_ARCHS "9.0a")
+endif()
+if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
+ list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0f")
+elseif(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
+ list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0a")
+endif()
+
+cuda_archs_loose_intersection(DEEPGEMM_ARCHS
+ "${DEEPGEMM_SUPPORT_ARCHS}" "${CUDA_ARCHS}")
+
+if(DEEPGEMM_ARCHS)
+ message(STATUS "DeepGEMM CUDA architectures: ${DEEPGEMM_ARCHS}")
+
+ find_package(CUDAToolkit REQUIRED)
+
+ #
+ # Build the _C pybind11 extension from DeepGEMM's C++ source.
+ # This is a CXX-only module — CUDA kernels are JIT-compiled at runtime.
+ #
+ Python_add_library(_deep_gemm_C MODULE WITH_SOABI
+ "${deepgemm_SOURCE_DIR}/csrc/python_api.cpp")
+
+ # The pybind11 module name must be _C to match DeepGEMM's Python imports.
+ set_target_properties(_deep_gemm_C PROPERTIES OUTPUT_NAME "_C")
+
+ target_compile_definitions(_deep_gemm_C PRIVATE
+ "-DTORCH_EXTENSION_NAME=_C")
+
+ target_include_directories(_deep_gemm_C PRIVATE
+ "${deepgemm_SOURCE_DIR}/csrc"
+ "${deepgemm_SOURCE_DIR}/deep_gemm/include"
+ "${deepgemm_SOURCE_DIR}/third-party/cutlass/include"
+ "${deepgemm_SOURCE_DIR}/third-party/cutlass/tools/util/include"
+ "${deepgemm_SOURCE_DIR}/third-party/fmt/include")
+
+ target_compile_options(_deep_gemm_C PRIVATE
+ $<$:-std=c++17>
+ $<$:-O3>
+ $<$:-Wno-psabi>
+ $<$:-Wno-deprecated-declarations>)
+
+ # torch_python is required because DeepGEMM uses pybind11 type casters
+ # for at::Tensor (via PYBIND11_MODULE), unlike vLLM's own extensions which
+ # use torch::Library custom ops.
+ find_library(TORCH_PYTHON_LIBRARY torch_python
+ PATHS "${TORCH_INSTALL_PREFIX}/lib"
+ REQUIRED)
+
+ target_link_libraries(_deep_gemm_C PRIVATE
+ torch ${TORCH_LIBRARIES} "${TORCH_PYTHON_LIBRARY}"
+ CUDA::cudart CUDA::nvrtc)
+
+ # Install the shared library into the vendored package directory
+ install(TARGETS _deep_gemm_C
+ LIBRARY DESTINATION vllm/third_party/deep_gemm
+ COMPONENT _deep_gemm_C)
+
+ #
+ # Vendor DeepGEMM Python package files
+ #
+ install(FILES
+ "${deepgemm_SOURCE_DIR}/deep_gemm/__init__.py"
+ DESTINATION vllm/third_party/deep_gemm
+ COMPONENT _deep_gemm_C)
+
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/deep_gemm/utils/"
+ DESTINATION vllm/third_party/deep_gemm/utils
+ COMPONENT _deep_gemm_C
+ FILES_MATCHING PATTERN "*.py")
+
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/deep_gemm/testing/"
+ DESTINATION vllm/third_party/deep_gemm/testing
+ COMPONENT _deep_gemm_C
+ FILES_MATCHING PATTERN "*.py")
+
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/deep_gemm/legacy/"
+ DESTINATION vllm/third_party/deep_gemm/legacy
+ COMPONENT _deep_gemm_C
+ FILES_MATCHING PATTERN "*.py")
+
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/deep_gemm/mega/"
+ DESTINATION vllm/third_party/deep_gemm/mega
+ COMPONENT _deep_gemm_C
+ FILES_MATCHING PATTERN "*.py")
+
+ # Generate envs.py (normally generated by DeepGEMM's setup.py build step)
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/deep_gemm_envs.py"
+ "# Pre-installed environment variables\npersistent_envs = dict()\n")
+ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/deep_gemm_envs.py"
+ DESTINATION vllm/third_party/deep_gemm
+ RENAME envs.py
+ COMPONENT _deep_gemm_C)
+
+ #
+ # Install include files needed for JIT compilation at runtime.
+ # The JIT compiler finds these relative to the package directory.
+ #
+
+ # DeepGEMM's own CUDA headers
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/deep_gemm/include/"
+ DESTINATION vllm/third_party/deep_gemm/include
+ COMPONENT _deep_gemm_C)
+
+ # CUTLASS and CuTe headers (vendored for JIT, separate from vLLM's CUTLASS)
+ install(DIRECTORY "${deepgemm_SOURCE_DIR}/third-party/cutlass/include/"
+ DESTINATION vllm/third_party/deep_gemm/include
+ COMPONENT _deep_gemm_C)
+
+else()
+ message(STATUS "DeepGEMM will not compile: "
+ "unsupported CUDA architecture ${CUDA_ARCHS}")
+ # Create empty target so setup.py doesn't fail on unsupported systems
+ add_custom_target(_deep_gemm_C)
+endif()
diff --git a/cmake/external_projects/flashmla.cmake b/cmake/external_projects/flashmla.cmake
index 0f16b9161fa3..65986df55012 100644
--- a/cmake/external_projects/flashmla.cmake
+++ b/cmake/external_projects/flashmla.cmake
@@ -19,7 +19,7 @@ else()
FetchContent_Declare(
flashmla
GIT_REPOSITORY https://github.com/vllm-project/FlashMLA
- GIT_TAG 692917b1cda61b93ac9ee2d846ec54e75afe87b1
+ GIT_TAG a6ec2ba7bd0a7dff98b3f4d3e6b52b159c48d78b
GIT_PROGRESS TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
diff --git a/cmake/external_projects/vllm_flash_attn.cmake b/cmake/external_projects/vllm_flash_attn.cmake
index 9414f5af3722..4aef7282134b 100644
--- a/cmake/external_projects/vllm_flash_attn.cmake
+++ b/cmake/external_projects/vllm_flash_attn.cmake
@@ -39,7 +39,7 @@ else()
FetchContent_Declare(
vllm-flash-attn
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
- GIT_TAG c0ec424fd8a546d0cbbf4bf050bbcfe837c55afb
+ GIT_TAG f5bc33cfc02c744d24a2e9d50e6db656de40611c
GIT_PROGRESS TRUE
# Don't share the vllm-flash-attn build between build types
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
@@ -87,18 +87,30 @@ endforeach()
#
add_custom_target(_vllm_fa4_cutedsl_C)
-# Copy flash_attn/cute directory (needed for FA4) and transform imports
-# The cute directory uses flash_attn.cute imports internally, which we replace
-# with vllm.vllm_flash_attn.cute to match our package structure.
-install(CODE "
- file(GLOB_RECURSE CUTE_PY_FILES \"${vllm-flash-attn_SOURCE_DIR}/flash_attn/cute/*.py\")
- foreach(SRC_FILE \${CUTE_PY_FILES})
- file(RELATIVE_PATH REL_PATH \"${vllm-flash-attn_SOURCE_DIR}/flash_attn/cute\" \${SRC_FILE})
- set(DST_FILE \"\${CMAKE_INSTALL_PREFIX}/vllm/vllm_flash_attn/cute/\${REL_PATH}\")
- get_filename_component(DST_DIR \${DST_FILE} DIRECTORY)
- file(MAKE_DIRECTORY \${DST_DIR})
- file(READ \${SRC_FILE} FILE_CONTENTS)
- string(REPLACE \"flash_attn.cute\" \"vllm.vllm_flash_attn.cute\" FILE_CONTENTS \"\${FILE_CONTENTS}\")
- file(WRITE \${DST_FILE} \"\${FILE_CONTENTS}\")
- endforeach()
-" COMPONENT _vllm_fa4_cutedsl_C)
+# Install flash_attn/cute directory (needed for FA4).
+# When using a local source dir (VLLM_FLASH_ATTN_SRC_DIR), create a symlink
+# so edits to cute-dsl Python files take effect immediately without rebuilding.
+# Otherwise, copy files and transform flash_attn.cute imports to
+# vllm.vllm_flash_attn.cute to match our package structure.
+if(VLLM_FLASH_ATTN_SRC_DIR)
+ install(CODE "
+ set(LINK_TARGET \"${vllm-flash-attn_SOURCE_DIR}/flash_attn/cute\")
+ set(LINK_NAME \"\${CMAKE_INSTALL_PREFIX}/vllm/vllm_flash_attn/cute\")
+ file(MAKE_DIRECTORY \"\${CMAKE_INSTALL_PREFIX}/vllm/vllm_flash_attn\")
+ file(REMOVE_RECURSE \"\${LINK_NAME}\")
+ file(CREATE_LINK \"\${LINK_TARGET}\" \"\${LINK_NAME}\" SYMBOLIC)
+ " COMPONENT _vllm_fa4_cutedsl_C)
+else()
+ install(CODE "
+ file(GLOB_RECURSE CUTE_PY_FILES \"${vllm-flash-attn_SOURCE_DIR}/flash_attn/cute/*.py\")
+ foreach(SRC_FILE \${CUTE_PY_FILES})
+ file(RELATIVE_PATH REL_PATH \"${vllm-flash-attn_SOURCE_DIR}/flash_attn/cute\" \${SRC_FILE})
+ set(DST_FILE \"\${CMAKE_INSTALL_PREFIX}/vllm/vllm_flash_attn/cute/\${REL_PATH}\")
+ get_filename_component(DST_DIR \${DST_FILE} DIRECTORY)
+ file(MAKE_DIRECTORY \${DST_DIR})
+ file(READ \${SRC_FILE} FILE_CONTENTS)
+ string(REPLACE \"flash_attn.cute\" \"vllm.vllm_flash_attn.cute\" FILE_CONTENTS \"\${FILE_CONTENTS}\")
+ file(WRITE \${DST_FILE} \"\${FILE_CONTENTS}\")
+ endforeach()
+ " COMPONENT _vllm_fa4_cutedsl_C)
+endif()
diff --git a/csrc/activation_kernels.cu b/csrc/activation_kernels.cu
index 758a77795553..303433392c32 100644
--- a/csrc/activation_kernels.cu
+++ b/csrc/activation_kernels.cu
@@ -11,29 +11,74 @@
namespace vllm {
template
+ bool act_first, bool HAS_CLAMP>
__device__ __forceinline__ scalar_t compute(const scalar_t& x,
- const scalar_t& y) {
- return act_first ? ACT_FN(x) * y : x * ACT_FN(y);
+ const scalar_t& y,
+ const float limit) {
+ if constexpr (act_first) {
+ scalar_t gate = x;
+ scalar_t up = y;
+ if constexpr (HAS_CLAMP) {
+ gate = (scalar_t)fminf((float)gate, limit);
+ up = (scalar_t)fmaxf(fminf((float)up, limit), -limit);
+ }
+ return ACT_FN(gate) * up;
+ } else {
+ scalar_t gate = x;
+ scalar_t up = y;
+ if constexpr (HAS_CLAMP) {
+ gate = (scalar_t)fmaxf(fminf((float)gate, limit), -limit);
+ up = (scalar_t)fminf((float)up, limit);
+ }
+ return gate * ACT_FN(up);
+ }
}
template
+ bool act_first, bool HAS_CLAMP>
__device__ __forceinline__ packed_t packed_compute(const packed_t& x,
- const packed_t& y) {
- return act_first ? packed_mul(PACKED_ACT_FN(x), y)
- : packed_mul(x, PACKED_ACT_FN(y));
+ const packed_t& y,
+ const float limit) {
+ if constexpr (act_first) {
+ packed_t gate = x;
+ packed_t up = y;
+ if constexpr (HAS_CLAMP) {
+ float2 g = cast_to_float2(gate);
+ float2 u = cast_to_float2(up);
+ g.x = fminf(g.x, limit);
+ g.y = fminf(g.y, limit);
+ u.x = fmaxf(fminf(u.x, limit), -limit);
+ u.y = fmaxf(fminf(u.y, limit), -limit);
+ gate = cast_to_packed(g);
+ up = cast_to_packed(u);
+ }
+ return packed_mul(PACKED_ACT_FN(gate), up);
+ } else {
+ packed_t gate = x;
+ packed_t up = y;
+ if constexpr (HAS_CLAMP) {
+ float2 g = cast_to_float2(gate);
+ float2 u = cast_to_float2(up);
+ g.x = fmaxf(fminf(g.x, limit), -limit);
+ g.y = fmaxf(fminf(g.y, limit), -limit);
+ u.x = fminf(u.x, limit);
+ u.y = fminf(u.y, limit);
+ gate = cast_to_packed(g);
+ up = cast_to_packed(u);
+ }
+ return packed_mul(gate, PACKED_ACT_FN(up));
+ }
}
// Activation and gating kernel template.
template
+ bool use_vec, bool HAS_CLAMP, bool use_256b = false>
__global__ void act_and_mul_kernel(
scalar_t* __restrict__ out, // [..., d]
const scalar_t* __restrict__ input, // [..., 2, d]
- const int d) {
+ const int d, const float limit) {
const scalar_t* x_ptr = input + blockIdx.x * 2 * d;
const scalar_t* y_ptr = x_ptr + d;
scalar_t* out_ptr = out + blockIdx.x * d;
@@ -58,8 +103,9 @@ __global__ void act_and_mul_kernel(
}
#pragma unroll
for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
- x.elts[j] = packed_compute(
- x.elts[j], y.elts[j]);
+ x.elts[j] =
+ packed_compute(
+ x.elts[j], y.elts[j], limit);
}
if constexpr (use_256b) {
st256(x, &out_vec[i]);
@@ -72,7 +118,8 @@ __global__ void act_and_mul_kernel(
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
const scalar_t x = VLLM_LDG(&x_ptr[idx]);
const scalar_t y = VLLM_LDG(&y_ptr[idx]);
- out_ptr[idx] = compute(x, y);
+ out_ptr[idx] =
+ compute(x, y, limit);
}
}
}
@@ -151,8 +198,11 @@ packed_gelu_tanh_kernel(const packed_t& val) {
// Launch activation and gating kernel.
// Use ACT_FIRST (bool) indicating whether to apply the activation function
-// first.
-#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST) \
+// first. HAS_CLAMP (bool) enables pre-activation clamping: gate input is
+// clamped (max only) and up input is clamped (both sides) before the
+// activation function is applied.
+#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST, \
+ HAS_CLAMP, LIMIT) \
auto dtype = input.scalar_type(); \
int d = input.size(-1) / 2; \
int64_t num_tokens = input.numel() / input.size(-1); \
@@ -177,8 +227,8 @@ packed_gelu_tanh_kernel(const packed_t& val) {
scalar_t, typename vllm::PackedTypeConverter::Type, \
KERNEL, \
PACKED_KERNEL::Type>, \
- ACT_FIRST, true, true><<>>( \
- out.data_ptr(), input.data_ptr(), d); \
+ ACT_FIRST, true, HAS_CLAMP, true><<>>( \
+ out.data_ptr(), input.data_ptr(), d, LIMIT); \
}); \
} else { \
VLLM_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] { \
@@ -186,8 +236,8 @@ packed_gelu_tanh_kernel(const packed_t& val) {
scalar_t, typename vllm::PackedTypeConverter::Type, \
KERNEL, \
PACKED_KERNEL::Type>, \
- ACT_FIRST, true, false><<>>( \
- out.data_ptr(), input.data_ptr(), d); \
+ ACT_FIRST, true, HAS_CLAMP, false><<>>( \
+ out.data_ptr(), input.data_ptr(), d, LIMIT); \
}); \
} \
} else { \
@@ -197,8 +247,8 @@ packed_gelu_tanh_kernel(const packed_t& val) {
scalar_t, typename vllm::PackedTypeConverter::Type, \
KERNEL, \
PACKED_KERNEL::Type>, \
- ACT_FIRST, false><<>>( \
- out.data_ptr(), input.data_ptr(), d); \
+ ACT_FIRST, false, HAS_CLAMP><<>>( \
+ out.data_ptr(), input.data_ptr(), d, LIMIT); \
}); \
}
@@ -206,7 +256,14 @@ void silu_and_mul(torch::Tensor& out, // [..., d]
torch::Tensor& input) // [..., 2 * d]
{
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
- true);
+ true, false, 0.0f);
+}
+
+void silu_and_mul_clamp(torch::Tensor& out, // [..., d]
+ torch::Tensor& input, // [..., 2 * d]
+ double limit) {
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
+ true, true, (float)limit);
}
void mul_and_silu(torch::Tensor& out, // [..., d]
@@ -215,21 +272,21 @@ void mul_and_silu(torch::Tensor& out, // [..., d]
// The difference between mul_and_silu and silu_and_mul is that mul_and_silu
// applies the silu to the latter half of the input.
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
- false);
+ false, false, 0.0f);
}
void gelu_and_mul(torch::Tensor& out, // [..., d]
torch::Tensor& input) // [..., 2 * d]
{
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, vllm::packed_gelu_kernel,
- true);
+ true, false, 0.0f);
}
void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
torch::Tensor& input) // [..., 2 * d]
{
- LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel,
- vllm::packed_gelu_tanh_kernel, true);
+ LAUNCH_ACTIVATION_GATE_KERNEL(
+ vllm::gelu_tanh_kernel, vllm::packed_gelu_tanh_kernel, true, false, 0.0f);
}
namespace vllm {
diff --git a/csrc/async_util.cuh b/csrc/async_util.cuh
new file mode 100644
index 000000000000..392d78c53fdb
--- /dev/null
+++ b/csrc/async_util.cuh
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace vllm {
+namespace cuda_async {
+
+__device__ __forceinline__ void cp_async_shared_global_16_cg(
+ void* smem_ptr, const void* glob_ptr) {
+#if defined(USE_ROCM)
+ *reinterpret_cast(smem_ptr) = *reinterpret_cast(glob_ptr);
+#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr));
+ asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n"
+ :
+ : "r"(smem), "l"(glob_ptr));
+#elif defined(__CUDA_ARCH__)
+ *reinterpret_cast(smem_ptr) = *reinterpret_cast(glob_ptr);
+#else
+ (void)smem_ptr;
+ (void)glob_ptr;
+#endif
+}
+
+__device__ __forceinline__ void cp_async_shared_global_ca(void* smem_ptr,
+ const void* glob_ptr,
+ int size_bytes) {
+#if defined(USE_ROCM)
+ if (size_bytes == 4) {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ } else if (size_bytes == 8) {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ } else {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ }
+#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr));
+ if (size_bytes == 4) {
+ asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n"
+ :
+ : "r"(smem), "l"(glob_ptr));
+ } else if (size_bytes == 8) {
+ asm volatile("cp.async.ca.shared.global [%0], [%1], 8;\n"
+ :
+ : "r"(smem), "l"(glob_ptr));
+ } else {
+ asm volatile("cp.async.ca.shared.global [%0], [%1], 16;\n"
+ :
+ : "r"(smem), "l"(glob_ptr));
+ }
+#elif defined(__CUDA_ARCH__)
+ if (size_bytes == 4) {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ } else if (size_bytes == 8) {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ } else {
+ *reinterpret_cast(smem_ptr) =
+ *reinterpret_cast(glob_ptr);
+ }
+#else
+ (void)smem_ptr;
+ (void)glob_ptr;
+ (void)size_bytes;
+#endif
+}
+
+__device__ __forceinline__ void cp_async_commit_group() {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 && !defined(USE_ROCM)
+ asm volatile("cp.async.commit_group;\n" ::);
+#endif
+}
+
+template
+__device__ __forceinline__ void cp_async_wait_group() {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 && !defined(USE_ROCM)
+ asm volatile("cp.async.wait_group %0;\n" : : "n"(n));
+#endif
+}
+
+} // namespace cuda_async
+} // namespace vllm
diff --git a/csrc/attention/dtype_fp8.cuh b/csrc/attention/dtype_fp8.cuh
index e714e321b0be..1afec3c39976 100644
--- a/csrc/attention/dtype_fp8.cuh
+++ b/csrc/attention/dtype_fp8.cuh
@@ -17,6 +17,22 @@ enum class Fp8KVCacheDataType {
kFp8E5M2 = 2,
};
+inline Fp8KVCacheDataType get_fp8_kv_cache_data_type(
+ const std::string& dtype_str) {
+ // dtype_str refers to CacheDType at vllm.config.cache.CacheDType
+ if (dtype_str == "auto" || dtype_str == "float16" ||
+ dtype_str == "bfloat16") {
+ // unquantized kv cache
+ return Fp8KVCacheDataType::kAuto;
+ } else if (dtype_str == "fp8" || dtype_str == "fp8_ds_mla" ||
+ dtype_str == "fp8_e4m3") {
+ return Fp8KVCacheDataType::kFp8E4M3;
+ } else if (dtype_str == "fp8_e5m2") {
+ return Fp8KVCacheDataType::kFp8E5M2;
+ }
+ TORCH_CHECK(false, "Unsupported fp8 kv cache data type: ", dtype_str);
+}
+
// fp8 vector types for quantization of kv cache
template <>
struct Vec {
diff --git a/csrc/cache_kernels.cu b/csrc/cache_kernels.cu
index a7e5fdabf648..7e456d32598b 100644
--- a/csrc/cache_kernels.cu
+++ b/csrc/cache_kernels.cu
@@ -104,37 +104,49 @@ void swap_blocks_batch(const torch::Tensor& src_ptrs,
static_assert(sizeof(CUdeviceptr) == sizeof(int64_t));
static_assert(sizeof(size_t) == sizeof(int64_t));
#if !defined(USE_ROCM) && defined(CUDA_VERSION) && CUDA_VERSION >= 12080
- CUmemcpyAttributes attr = {};
- attr.srcAccessOrder = CU_MEMCPY_SRC_ACCESS_ORDER_STREAM;
- size_t attrs_idx = 0;
- #if defined(CUDA_VERSION) && CUDA_VERSION >= 13000
- CUresult result = cuMemcpyBatchAsync(
- reinterpret_cast(dst_data),
- reinterpret_cast(src_data),
- reinterpret_cast(size_data), static_cast(n), &attr,
- &attrs_idx, 1, static_cast(stream));
- TORCH_CHECK(result == CUDA_SUCCESS, "cuMemcpyBatchAsync failed with error ",
- result);
- #else
- size_t fail_idx = 0;
- CUresult result = cuMemcpyBatchAsync(
- reinterpret_cast(dst_data),
- reinterpret_cast(src_data),
- reinterpret_cast(size_data), static_cast(n), &attr,
- &attrs_idx, 1, &fail_idx, static_cast(stream));
- TORCH_CHECK(result == CUDA_SUCCESS, "cuMemcpyBatchAsync failed at index ",
- fail_idx, " with error ", result);
- #endif
-#else
- // Fallback for CUDA < 12.8 and ROCm: individual async copies.
- // cudaMemcpyDefault lets the driver infer direction from pointer types.
- for (int64_t i = 0; i < n; i++) {
- cudaMemcpyAsync(reinterpret_cast(dst_data[i]),
- reinterpret_cast(src_data[i]),
- static_cast(size_data[i]), cudaMemcpyDefault,
- stream);
- }
+ // Resolve cuMemcpyBatchAsync at runtime via cuGetProcAddress so that
+ // binaries compiled with CUDA 12.8+ still work on older drivers, and
+ // we avoid the CUDA 13.0 header remapping (#define to _v2 signature).
+ // The function pointer is cached after the first call.
+ using BatchFn =
+ CUresult (*)(CUdeviceptr*, CUdeviceptr*, size_t*, size_t,
+ CUmemcpyAttributes*, size_t*, size_t, size_t*, CUstream);
+ static BatchFn batch_fn = []() -> BatchFn {
+ CUdriverProcAddressQueryResult sym_status;
+ void* fn_ptr = nullptr;
+ CUresult res = cuGetProcAddress("cuMemcpyBatchAsync", &fn_ptr, 12080,
+ CU_GET_PROC_ADDRESS_DEFAULT, &sym_status);
+ if (res != CUDA_SUCCESS || fn_ptr == nullptr) {
+ return nullptr;
+ }
+ return reinterpret_cast(fn_ptr);
+ }();
+
+ if (batch_fn != nullptr) {
+ CUmemcpyAttributes attr = {};
+ attr.srcAccessOrder = CU_MEMCPY_SRC_ACCESS_ORDER_STREAM;
+ size_t attrs_idx = 0;
+ size_t fail_idx = 0;
+ CUresult result = batch_fn(reinterpret_cast(dst_data),
+ reinterpret_cast(src_data),
+ reinterpret_cast(size_data),
+ static_cast(n), &attr, &attrs_idx, 1,
+ &fail_idx, static_cast(stream));
+ TORCH_CHECK(result == CUDA_SUCCESS, "cuMemcpyBatchAsync failed at index ",
+ fail_idx, " with error ", result);
+ } else
#endif
+ {
+ // Fallback for CUDA < 12.8, older drivers, and ROCm:
+ // individual async copies.
+ // cudaMemcpyDefault lets the driver infer direction from pointer types.
+ for (int64_t i = 0; i < n; i++) {
+ cudaMemcpyAsync(reinterpret_cast(dst_data[i]),
+ reinterpret_cast(src_data[i]),
+ static_cast(size_data[i]), cudaMemcpyDefault,
+ stream);
+ }
+ }
}
namespace vllm {
@@ -587,6 +599,11 @@ __global__ void cp_gather_indexer_k_quant_cache_kernel(
const int head_idx = (blockIdx.y * blockDim.x + threadIdx.x) * VEC_SIZE;
// Find batch index within a block
__shared__ int batch_idx[BLOCK_Y_SIZE];
+ if (threadIdx.x == 0) {
+ batch_idx[threadIdx.y] = -1;
+ }
+ __syncthreads();
+
for (int iter = 0; iter < cuda_utils::ceil_div(batch_size, int(blockDim.x));
iter++) {
int tid = iter * blockDim.x + threadIdx.x;
@@ -599,16 +616,18 @@ __global__ void cp_gather_indexer_k_quant_cache_kernel(
}
}
-#ifndef USE_ROCM
- __syncwarp();
-#endif
+ __syncthreads();
- if (head_idx >= head_dim || token_idx >= num_tokens) {
+ // num_tokens may be an allocation upper bound when Python avoids a D2H sync.
+ // Only tokens covered by the exact device-side cu_seq_lens are valid to
+ // gather.
+ const int batch = batch_idx[threadIdx.y];
+ if (head_idx >= head_dim || token_idx >= num_tokens || batch < 0) {
return;
}
- const int inbatch_seq_idx = token_idx - cu_seq_lens[batch_idx[threadIdx.y]];
- const int block_idx = block_table[batch_idx[threadIdx.y] * num_blocks +
- inbatch_seq_idx / cache_block_size];
+ const int inbatch_seq_idx = token_idx - cu_seq_lens[batch];
+ const int block_idx =
+ block_table[batch * num_blocks + inbatch_seq_idx / cache_block_size];
const int64_t src_block_offset = block_idx * block_stride;
const int64_t cache_inblock_offset =
(inbatch_seq_idx % cache_block_size) * head_dim + head_idx;
@@ -712,6 +731,28 @@ void reshape_and_cache_flash(
int num_tokens = slot_mapping.size(0);
int num_heads = key.size(1);
int head_size = key.size(2);
+
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
+
+ if (kv_cache_dtype == "nvfp4") {
+#if defined(ENABLE_NVFP4_SM100) || defined(ENABLE_NVFP4_SM120)
+ // NVFP4 dispatch is compiled separately for SM100+.
+ extern void reshape_and_cache_nvfp4_dispatch(
+ torch::Tensor & key, torch::Tensor & value, torch::Tensor & key_cache,
+ torch::Tensor & value_cache, torch::Tensor & slot_mapping,
+ torch::Tensor & k_scale, torch::Tensor & v_scale);
+ reshape_and_cache_nvfp4_dispatch(key, value, key_cache, value_cache,
+ slot_mapping, k_scale, v_scale);
+ return;
+#else
+ TORCH_CHECK(false,
+ "NVFP4 KV cache requires SM100+ (Blackwell). "
+ "Please rebuild vllm with a Blackwell-compatible CUDA target.");
+#endif
+ }
+
+ // Original FP8/auto path.
int block_size = key_cache.size(1);
int64_t key_stride = key.stride(0);
@@ -729,8 +770,6 @@ void reshape_and_cache_flash(
dim3 grid(num_tokens);
dim3 block(std::min(num_heads * head_size, 512));
- const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype,
CALL_RESHAPE_AND_CACHE_FLASH);
@@ -1458,6 +1497,9 @@ void concat_mla_q(torch::Tensor& ql_nope, // [num_tokens, num_heads, nope_dim]
TORCH_CHECK(ql_nope.stride(2) == 1, "ql_nope must have stride 1 in dim 2");
TORCH_CHECK(q_pe.stride(2) == 1, "q_pe must have stride 1 in dim 2");
TORCH_CHECK(q_out.stride(2) == 1, "q_out must have stride 1 in dim 2");
+ TORCH_CHECK(ql_nope.scalar_type() == at::ScalarType::Half ||
+ ql_nope.scalar_type() == at::ScalarType::BFloat16,
+ "ql_nope must be float16 or bfloat16 dtype");
if (num_tokens == 0) return;
@@ -1469,7 +1511,7 @@ void concat_mla_q(torch::Tensor& ql_nope, // [num_tokens, num_heads, nope_dim]
const at::cuda::OptionalCUDAGuard device_guard(device_of(ql_nope));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
- VLLM_DISPATCH_FLOATING_TYPES(ql_nope.scalar_type(), "concat_mla_q", [&] {
+ VLLM_DISPATCH_HALF_TYPES(ql_nope.scalar_type(), "concat_mla_q", [&] {
vllm::ConcatMLAQKernel<<>>(
q_out.data_ptr(), ql_nope.data_ptr(),
q_pe.data_ptr(), num_tokens, num_heads, q_out.stride(0),
diff --git a/csrc/cpu/activation_lut_bf16.cpp b/csrc/cpu/activation_lut_bf16.cpp
new file mode 100644
index 000000000000..0ff2567e1ee8
--- /dev/null
+++ b/csrc/cpu/activation_lut_bf16.cpp
@@ -0,0 +1,71 @@
+#include "cpu_types.hpp"
+
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+constexpr uint32_t ActivationLutSize = 1u << 16;
+
+at::Tensor gelu_reference(const at::Tensor& x) { return at::gelu(x, "none"); }
+
+void maybe_init_activation_lut_bf16(
+ uint16_t* lut, std::once_flag& once,
+ at::Tensor (*activation)(const at::Tensor&)) {
+ std::call_once(once, [&]() {
+ auto lut_input =
+ at::empty({static_cast(ActivationLutSize)},
+ at::TensorOptions().device(at::kCPU).dtype(at::kFloat));
+ auto* lut_input_ptr = lut_input.data_ptr();
+#pragma omp parallel for
+ for (uint32_t i = 0; i < ActivationLutSize; ++i) {
+ lut_input_ptr[i] = c10::detail::f32_from_bits(static_cast(i));
+ }
+
+ auto lut_output = activation(lut_input);
+ const auto* lut_output_ptr = lut_output.data_ptr();
+#pragma omp parallel for
+ for (uint32_t i = 0; i < ActivationLutSize; ++i) {
+ lut[i] = c10::detail::round_to_nearest_even(lut_output_ptr[i]);
+ }
+ });
+}
+
+void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
+ const uint16_t* lut, const char* op_name) {
+ TORCH_CHECK(input.scalar_type() == at::kBFloat16, op_name,
+ ": input must be bfloat16");
+ TORCH_CHECK(out.scalar_type() == at::kBFloat16, op_name,
+ ": out must be bfloat16");
+ TORCH_CHECK(input.is_contiguous(), op_name, ": input must be contiguous");
+ TORCH_CHECK(out.is_contiguous(), op_name, ": out must be contiguous");
+
+ const auto* src =
+ reinterpret_cast(input.data_ptr());
+ auto* dst = reinterpret_cast(out.data_ptr());
+ const int64_t n = input.numel();
+
+ CPU_KERNEL_GUARD_IN(activation_lut_bf16_impl)
+#pragma omp parallel for
+ for (int64_t i = 0; i < n; ++i) {
+ dst[i] = lut[src[i]];
+ }
+ CPU_KERNEL_GUARD_OUT(activation_lut_bf16_impl)
+}
+
+void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
+ const std::string& activation) {
+ if (activation == "gelu") {
+ static std::array lut{};
+ static std::once_flag once;
+ maybe_init_activation_lut_bf16(lut.data(), once, gelu_reference);
+ activation_lut_bf16(out, input, lut.data(), "gelu_lut");
+ return;
+ }
+
+ TORCH_CHECK(false, "Unsupported activation: ", activation);
+}
diff --git a/csrc/cpu/cpu_arch_macros.h b/csrc/cpu/cpu_arch_macros.h
index c73b62ecdec9..9be45a3efcef 100644
--- a/csrc/cpu/cpu_arch_macros.h
+++ b/csrc/cpu/cpu_arch_macros.h
@@ -61,8 +61,23 @@
#endif
#ifdef __aarch64__
- // Implementation copied from Arm Optimized Routines (expf AdvSIMD)
+ // Implementation of neon_expf copied from Arm Optimized Routines (expf
+ // AdvSIMD)
// https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/expf.c
+ //
+ // Additional fast exponential intended for cases where outputs will be
+ // downcasted to FP16 / BF16 (e.g. attention softmax). Accurate within 1 ULP
+ // for FP16 Accurate within 1 ULP for BF16 for inputs in [-87.683, 88.376] &
+ // clamps inputs outside this range to 0 / inf. Implementation is similar to
+ // exp_u20, but:
+ // - uses a third degree polynomial approximation for exp(r) instead of a
+ // fifth degree one, with coefficients re-tuned.
+ // - does not split natural log (ln) into high / low parts
+ // - clamps exp(x) to 0 for x < -87.683113f and inf for x > 88.3762589f
+ // exp(x) = 2^n (exp(r))
+ // r = x - n*ln2, with n = round(x/ln2)
+ // exp(r) ~ poly(r) = 1 + r + r^2 * (c3 + c2 * r)
+ // n = round(x / ln2), r = x - n*ln2
#include
#define DEFINE_FAST_EXP \
const float32x4_t inv_ln2 = vdupq_n_f32(0x1.715476p+0f); \
@@ -106,7 +121,38 @@
result.val[2] = neon_expf(vec.reg.val[2]); \
result.val[3] = neon_expf(vec.reg.val[3]); \
return vec_op::FP32Vec16(result); \
- };
+ }; \
+ const float32x4_t lower_bound = vdupq_n_f32(-0x1.5ebb82p+6f); \
+ const float32x4_t upper_bound = vdupq_n_f32(0x1.61814ap+6f); \
+ constexpr float ln2 = 0x1.62e43p-1f; \
+ constexpr float f_c2 = 0x1.5592ecp-3f; \
+ const float32x4_t f_c3 = vdupq_n_f32(0x1.017d34p-1f); \
+ auto neon_expf_f16 = [&](float32x4_t values) __attribute__(( \
+ always_inline)) { \
+ const uint32x4_t lt_lower = vcltq_f32(values, lower_bound); \
+ const uint32x4_t gt_upper = vcgtq_f32(values, upper_bound); \
+ float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \
+ float32x4_t r = vfmsq_n_f32(values, n, ln2); \
+ uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \
+ float32x4_t r2 = vmulq_f32(r, r); \
+ float32x4_t q = vfmaq_n_f32(f_c3, r, f_c2); \
+ float32x4_t s = vaddq_f32(vdupq_n_f32(1.0f), r); \
+ float32x4_t p = vfmaq_f32(s, q, r2); \
+ float32x4_t y = \
+ vreinterpretq_f32_u32(vaddq_u32(vreinterpretq_u32_f32(p), e)); \
+ y = vbslq_f32(lt_lower, vdupq_n_f32(0.0f), y); \
+ y = vbslq_f32(gt_upper, vdupq_n_f32(INFINITY), y); \
+ return y; \
+ }; \
+ auto fast_exp_f16 = [&](const vec_op::FP32Vec16& vec) \
+ __attribute__((always_inline)) { \
+ float32x4x4_t result; \
+ result.val[0] = neon_expf_f16(vec.reg.val[0]); \
+ result.val[1] = neon_expf_f16(vec.reg.val[1]); \
+ result.val[2] = neon_expf_f16(vec.reg.val[2]); \
+ result.val[3] = neon_expf_f16(vec.reg.val[3]); \
+ return vec_op::FP32Vec16(result); \
+ };
#endif // __aarch64__
diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp
index c15799fa950d..c1974bfd0a51 100644
--- a/csrc/cpu/cpu_attn_impl.hpp
+++ b/csrc/cpu/cpu_attn_impl.hpp
@@ -147,6 +147,9 @@ struct AttentionMetadata {
case ISA::NEON:
ss << "NEON, ";
break;
+ case ISA::VXE:
+ ss << "VXE, ";
+ break;
}
ss << "workitem_group_num: " << workitem_group_num
<< ", reduction_item_num: " << reduction_item_num
@@ -1149,7 +1152,11 @@ class AttentionMainLoop {
bool use_sink) {
#ifdef DEFINE_FAST_EXP
DEFINE_FAST_EXP
+ bool constexpr IsReducedPrecision =
+ std::is_same_v ||
+ std::is_same_v;
#endif
+
using prob_buffer_vec_t = typename VecTypeTrait::vec_t;
static_assert(sizeof(prob_buffer_t) <= sizeof(logits_buffer_t));
@@ -1198,8 +1205,17 @@ class AttentionMainLoop {
vec = vec - max_vec;
// compute exp
-#ifdef DEFINE_FAST_EXP
- vec = fast_exp(vec);
+
+#if defined(DEFINE_FAST_EXP)
+ #ifdef __aarch64__
+ if constexpr (IsReducedPrecision) {
+ vec = fast_exp_f16(vec);
+ } else
+ #endif
+ {
+ vec = fast_exp(vec);
+ }
+
prob_buffer_vec_t output_vec(vec);
output_vec.save(curr_prob_buffer_iter);
#else
@@ -1255,7 +1271,11 @@ class AttentionMainLoop {
int32_t kv_tile_token_num, float softcap_scale) {
#ifdef DEFINE_FAST_EXP
DEFINE_FAST_EXP
+ bool constexpr IsReducedPrecision =
+ std::is_same_v ||
+ std::is_same_v;
#endif
+
float inv_softcap_scale = 1.0 / softcap_scale;
vec_op::FP32Vec16 softcap_scale_vec(softcap_scale);
vec_op::FP32Vec16 inv_softcap_scale_vec(inv_softcap_scale);
@@ -1269,8 +1289,15 @@ class AttentionMainLoop {
vec_op::FP32Vec16 vec(curr_logits_buffer_iter);
vec = vec * inv_softcap_scale_vec;
-#ifdef DEFINE_FAST_EXP
- vec = fast_exp(vec);
+#if defined(DEFINE_FAST_EXP)
+ #ifdef __aarch64__
+ if constexpr (IsReducedPrecision) {
+ vec = fast_exp_f16(vec);
+ } else
+ #endif
+ {
+ vec = fast_exp(vec);
+ }
vec_op::FP32Vec16 inv_vec = ones_vec / vec;
vec = (vec - inv_vec) / (vec + inv_vec);
#else
diff --git a/csrc/cpu/cpu_types_riscv.hpp b/csrc/cpu/cpu_types_riscv.hpp
index 910ee5c11331..e617d98dd002 100644
--- a/csrc/cpu/cpu_types_riscv.hpp
+++ b/csrc/cpu/cpu_types_riscv.hpp
@@ -1,832 +1,25 @@
#ifndef CPU_TYPES_RISCV_HPP
#define CPU_TYPES_RISCV_HPP
-#include
-#include
-#include
-#include
-#include
-#include
-#include
+// RISC-V Vector (RVV) CPU type definitions for vLLM.
+//
+// Supports multiple VLENs via compile-time dispatch. The compiler defines
+// __riscv_v_min_vlen from the zvlb extension in -march. The defs header
+// maps VLEN to the correct LMUL suffixes, and the impl header provides
+// VLEN-independent class implementations.
+//
+// To add support for a new VLEN, add the LMUL mapping in
+// cpu_types_riscv_defs.hpp (the impl header needs no changes).
-// ============================================================================
-// Vector Register Type Definitions (VLEN=128 bits)
-// ============================================================================
-
-typedef vfloat16m1_t fixed_vfloat16m1_t
- __attribute__((riscv_rvv_vector_bits(128)));
-typedef vfloat16m2_t fixed_vfloat16m2_t
- __attribute__((riscv_rvv_vector_bits(256)));
-
-typedef vfloat32m1_t fixed_vfloat32m1_t
- __attribute__((riscv_rvv_vector_bits(128)));
-typedef vfloat32m2_t fixed_vfloat32m2_t
- __attribute__((riscv_rvv_vector_bits(256)));
-typedef vfloat32m4_t fixed_vfloat32m4_t
- __attribute__((riscv_rvv_vector_bits(512)));
-typedef vfloat32m8_t fixed_vfloat32m8_t
- __attribute__((riscv_rvv_vector_bits(1024)));
-
-typedef vint32m2_t fixed_vint32m2_t __attribute__((riscv_rvv_vector_bits(256)));
-typedef vint32m4_t fixed_vint32m4_t __attribute__((riscv_rvv_vector_bits(512)));
-
-typedef vuint16m1_t fixed_vuint16m1_t
- __attribute__((riscv_rvv_vector_bits(128)));
-typedef vuint16m2_t fixed_vuint16m2_t
- __attribute__((riscv_rvv_vector_bits(256)));
-typedef vuint16m4_t fixed_vuint16m4_t
- __attribute__((riscv_rvv_vector_bits(512)));
-
-#ifdef RISCV_BF16_SUPPORT
-typedef vbfloat16m1_t fixed_vbfloat16m1_t
- __attribute__((riscv_rvv_vector_bits(128)));
-typedef vbfloat16m2_t fixed_vbfloat16m2_t
- __attribute__((riscv_rvv_vector_bits(256)));
-typedef vbfloat16m4_t fixed_vbfloat16m4_t
- __attribute__((riscv_rvv_vector_bits(512)));
-#endif
-
-namespace vec_op {
-
-#ifdef RISCV_BF16_SUPPORT
- #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
-#else
- #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
+#ifndef __riscv_vector
+ #error "cpu_types_riscv.hpp included in a non-RVV translation unit"
#endif
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
- AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
-
-#define FORCE_INLINE __attribute__((always_inline)) inline
-
-namespace {
-template
-constexpr void unroll_loop_item(std::integer_sequence, F&& f) {
- (f(std::integral_constant{}), ...);
-};
-} // namespace
-
-template >>
-constexpr void unroll_loop(F&& f) {
- unroll_loop_item(std::make_integer_sequence{}, std::forward(f));
-}
-
-template
-struct Vec {
- constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; };
-};
-
-struct FP32Vec8;
-struct FP32Vec16;
-
-// ============================================================================
-// FP16 Implementation
-// ============================================================================
-
-struct FP16Vec8 : public Vec {
- constexpr static int VEC_ELEM_NUM = 8;
- fixed_vfloat16m1_t reg;
-
- explicit FP16Vec8(const void* ptr)
- : reg(__riscv_vle16_v_f16m1(static_cast(ptr),
- VEC_ELEM_NUM)) {};
-
- explicit FP16Vec8(const FP32Vec8&);
-
- void save(void* ptr) const {
- __riscv_vse16_v_f16m1(static_cast<_Float16*>(ptr), reg, VEC_ELEM_NUM);
- }
- void save(void* ptr, int elem_num) const {
- __riscv_vse16_v_f16m1(static_cast<_Float16*>(ptr), reg, elem_num);
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(_Float16);
- __riscv_vsse16_v_f16m1(static_cast<_Float16*>(ptr), byte_stride, reg,
- VEC_ELEM_NUM);
- }
-};
-
-struct FP16Vec16 : public Vec {
- constexpr static int VEC_ELEM_NUM = 16;
- fixed_vfloat16m2_t reg;
-
- explicit FP16Vec16(const void* ptr)
- : reg(__riscv_vle16_v_f16m2(static_cast(ptr),
- VEC_ELEM_NUM)) {};
-
- explicit FP16Vec16(const FP32Vec16& vec);
-
- void save(void* ptr) const {
- __riscv_vse16_v_f16m2(static_cast<_Float16*>(ptr), reg, VEC_ELEM_NUM);
- }
- void save(void* ptr, int elem_num) const {
- __riscv_vse16_v_f16m2(static_cast<_Float16*>(ptr), reg, elem_num);
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(_Float16);
- __riscv_vsse16_v_f16m2(static_cast<_Float16*>(ptr), byte_stride, reg,
- VEC_ELEM_NUM);
- }
-};
-
-// ============================================================================
-// BF16 Implementation
-// ============================================================================
-
-#ifdef RISCV_BF16_SUPPORT
-
-FORCE_INLINE fixed_vuint16m1_t bf16_to_u16(fixed_vbfloat16m1_t v) {
- return __riscv_vreinterpret_v_bf16m1_u16m1(v);
-}
-FORCE_INLINE fixed_vuint16m2_t bf16_to_u16(fixed_vbfloat16m2_t v) {
- return __riscv_vreinterpret_v_bf16m2_u16m2(v);
-}
-FORCE_INLINE fixed_vuint16m4_t bf16_to_u16(fixed_vbfloat16m4_t v) {
- return __riscv_vreinterpret_v_bf16m4_u16m4(v);
-}
-
-struct BF16Vec8 : public Vec {
- constexpr static int VEC_ELEM_NUM = 8;
- fixed_vbfloat16m1_t reg;
-
- explicit BF16Vec8(const void* ptr)
- : reg(__riscv_vreinterpret_v_u16m1_bf16m1(__riscv_vle16_v_u16m1(
- reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
-
- explicit BF16Vec8(fixed_vbfloat16m1_t data) : reg(data) {};
- explicit BF16Vec8(const FP32Vec8&);
-
- void save(void* ptr) const {
- __riscv_vse16_v_u16m1(reinterpret_cast(ptr), bf16_to_u16(reg),
- VEC_ELEM_NUM);
- }
- void save(void* ptr, int elem_num) const {
- __riscv_vse16_v_u16m1(reinterpret_cast(ptr), bf16_to_u16(reg),
- elem_num);
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- __riscv_vsse16_v_u16m1(reinterpret_cast(ptr), byte_stride,
- bf16_to_u16(reg), VEC_ELEM_NUM);
- }
-};
-
-struct BF16Vec16 : public Vec {
- constexpr static int VEC_ELEM_NUM = 16;
- fixed_vbfloat16m2_t reg;
-
- explicit BF16Vec16(const void* ptr)
- : reg(__riscv_vreinterpret_v_u16m2_bf16m2(__riscv_vle16_v_u16m2(
- reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
-
- explicit BF16Vec16(fixed_vbfloat16m2_t data) : reg(data) {};
- explicit BF16Vec16(const FP32Vec16&);
-
- void save(void* ptr) const {
- __riscv_vse16_v_u16m2(reinterpret_cast(ptr), bf16_to_u16(reg),
- VEC_ELEM_NUM);
- }
- void save(void* ptr, int elem_num) const {
- __riscv_vse16_v_u16m2(reinterpret_cast(ptr), bf16_to_u16(reg),
- elem_num);
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- __riscv_vsse16_v_u16m2(reinterpret_cast(ptr), byte_stride,
- bf16_to_u16(reg), VEC_ELEM_NUM);
- }
-};
-
-struct BF16Vec32 : public Vec {
- constexpr static int VEC_ELEM_NUM = 32;
- fixed_vbfloat16m4_t reg;
-
- explicit BF16Vec32(const void* ptr)
- : reg(__riscv_vreinterpret_v_u16m4_bf16m4(__riscv_vle16_v_u16m4(
- reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
-
- explicit BF16Vec32(fixed_vbfloat16m4_t data) : reg(data) {};
-
- explicit BF16Vec32(const BF16Vec8& v) {
- fixed_vuint16m1_t u16_val = bf16_to_u16(v.reg);
- fixed_vuint16m4_t u16_combined =
- __riscv_vcreate_v_u16m1_u16m4(u16_val, u16_val, u16_val, u16_val);
- reg = __riscv_vreinterpret_v_u16m4_bf16m4(u16_combined);
- };
-
- void save(void* ptr) const {
- __riscv_vse16_v_u16m4(reinterpret_cast(ptr), bf16_to_u16(reg),
- VEC_ELEM_NUM);
- }
- void save(void* ptr, int elem_num) const {
- __riscv_vse16_v_u16m4(reinterpret_cast(ptr), bf16_to_u16(reg),
- elem_num);
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- __riscv_vsse16_v_u16m4(reinterpret_cast(ptr), byte_stride,
- bf16_to_u16(reg), VEC_ELEM_NUM);
- }
-};
-
-#else
-// ============================================================================
-// BF16 Fallback Implementation (FP32 Simulation)
-// ============================================================================
-
-struct BF16Vec8 : public Vec {
- constexpr static int VEC_ELEM_NUM = 8;
- fixed_vfloat32m2_t reg_fp32;
- explicit BF16Vec8(const void* ptr) {
- const uint16_t* u16 = static_cast(ptr);
- float tmp[8];
- for (int i = 0; i < 8; ++i) {
- uint32_t v = static_cast(u16[i]) << 16;
- std::memcpy(&tmp[i], &v, 4);
- }
- reg_fp32 = __riscv_vle32_v_f32m2(tmp, 8);
- }
- explicit BF16Vec8(const FP32Vec8&);
- void save(void* ptr) const {
- float tmp[8];
- __riscv_vse32_v_f32m2(tmp, reg_fp32, 8);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < 8; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
- void save(void* ptr, int elem_num) const {
- float tmp[8];
- __riscv_vse32_v_f32m2(tmp, reg_fp32, 8);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < elem_num; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- float tmp[8];
- __riscv_vse32_v_f32m2(tmp, reg_fp32, 8);
- uint8_t* u8 = static_cast(ptr);
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- for (int i = 0; i < 8; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- uint16_t val = static_cast(v >> 16);
- *reinterpret_cast(u8 + i * byte_stride) = val;
- }
- }
-};
-
-struct BF16Vec16 : public Vec {
- constexpr static int VEC_ELEM_NUM = 16;
- fixed_vfloat32m4_t reg_fp32;
- explicit BF16Vec16(const void* ptr) {
- const uint16_t* u16 = static_cast(ptr);
- float tmp[16];
- for (int i = 0; i < 16; ++i) {
- uint32_t v = static_cast(u16[i]) << 16;
- std::memcpy(&tmp[i], &v, 4);
- }
- reg_fp32 = __riscv_vle32_v_f32m4(tmp, 16);
- }
- explicit BF16Vec16(const FP32Vec16&);
- void save(void* ptr) const {
- float tmp[16];
- __riscv_vse32_v_f32m4(tmp, reg_fp32, 16);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < 16; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
- void save(void* ptr, int elem_num) const {
- float tmp[16];
- __riscv_vse32_v_f32m4(tmp, reg_fp32, 16);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < elem_num; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
- void save_strided(void* ptr, ptrdiff_t stride) const {
- float tmp[16];
- __riscv_vse32_v_f32m4(tmp, reg_fp32, 16);
- uint8_t* u8 = static_cast(ptr);
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- for (int i = 0; i < 16; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- uint16_t val = static_cast(v >> 16);
- *reinterpret_cast(u8 + i * byte_stride) = val;
- }
- }
-};
-
-struct BF16Vec32 : public Vec {
- constexpr static int VEC_ELEM_NUM = 32;
- fixed_vfloat32m8_t reg_fp32;
-
- explicit BF16Vec32(const void* ptr) {
- const uint16_t* u16 = static_cast(ptr);
- float tmp[32];
- for (int i = 0; i < 32; ++i) {
- uint32_t v = static_cast(u16[i]) << 16;
- std::memcpy(&tmp[i], &v, 4);
- }
- reg_fp32 = __riscv_vle32_v_f32m8(tmp, 32);
- }
-
- explicit BF16Vec32(const BF16Vec8& v) {
- float tmp_small[8];
- __riscv_vse32_v_f32m2(tmp_small, v.reg_fp32, 8);
- float tmp_large[32];
- for (int i = 0; i < 4; ++i) {
- std::memcpy(tmp_large + (i * 8), tmp_small, 8 * sizeof(float));
- }
- reg_fp32 = __riscv_vle32_v_f32m8(tmp_large, 32);
- }
-
- void save(void* ptr) const {
- float tmp[32];
- __riscv_vse32_v_f32m8(tmp, reg_fp32, 32);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < 32; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
-
- void save(void* ptr, int elem_num) const {
- float tmp[32];
- __riscv_vse32_v_f32m8(tmp, reg_fp32, 32);
- uint16_t* u16 = static_cast(ptr);
- for (int i = 0; i < elem_num; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- u16[i] = static_cast(v >> 16);
- }
- }
-
- void save_strided(void* ptr, ptrdiff_t stride) const {
- float tmp[32];
- __riscv_vse32_v_f32m8(tmp, reg_fp32, 32);
- uint8_t* u8 = static_cast(ptr);
- ptrdiff_t byte_stride = stride * sizeof(uint16_t);
- for (int i = 0; i < 32; ++i) {
- uint32_t v;
- std::memcpy(&v, &tmp[i], 4);
- uint16_t val = static_cast(v >> 16);
- *reinterpret_cast(u8 + i * byte_stride) = val;
- }
- }
-};
-#endif
-
-// ============================================================================
-// FP32 Implementation
-// ============================================================================
-
-struct FP32Vec4 : public Vec {
- constexpr static int VEC_ELEM_NUM = 4;
- fixed_vfloat32m1_t reg;
- explicit FP32Vec4(float v) : reg(__riscv_vfmv_v_f_f32m1(v, VEC_ELEM_NUM)) {};
- explicit FP32Vec4() : reg(__riscv_vfmv_v_f_f32m1(0.0f, VEC_ELEM_NUM)) {};
- explicit FP32Vec4(const float* ptr)
- : reg(__riscv_vle32_v_f32m1(ptr, VEC_ELEM_NUM)) {};
- explicit FP32Vec4(fixed_vfloat32m1_t data) : reg(data) {};
- explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {};
- void save(float* ptr) const { __riscv_vse32_v_f32m1(ptr, reg, VEC_ELEM_NUM); }
- void save(float* ptr, int elem_num) const {
- __riscv_vse32_v_f32m1(ptr, reg, elem_num);
- }
-};
-
-struct FP32Vec8 : public Vec {
- constexpr static int VEC_ELEM_NUM = 8;
- fixed_vfloat32m2_t reg;
-
- explicit FP32Vec8(float v) : reg(__riscv_vfmv_v_f_f32m2(v, VEC_ELEM_NUM)) {};
- explicit FP32Vec8() : reg(__riscv_vfmv_v_f_f32m2(0.0f, VEC_ELEM_NUM)) {};
- explicit FP32Vec8(const float* ptr)
- : reg(__riscv_vle32_v_f32m2(ptr, VEC_ELEM_NUM)) {};
- explicit FP32Vec8(fixed_vfloat32m2_t data) : reg(data) {};
- explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {};
- explicit FP32Vec8(const FP16Vec8& v)
- : reg(__riscv_vfwcvt_f_f_v_f32m2(v.reg, VEC_ELEM_NUM)) {};
- explicit FP32Vec8(fixed_vfloat16m1_t v)
- : reg(__riscv_vfwcvt_f_f_v_f32m2(v, VEC_ELEM_NUM)) {};
-
-#ifdef RISCV_BF16_SUPPORT
- explicit FP32Vec8(fixed_vbfloat16m1_t v)
- : reg(__riscv_vfwcvtbf16_f_f_v_f32m2(v, VEC_ELEM_NUM)) {};
- explicit FP32Vec8(const BF16Vec8& v)
- : reg(__riscv_vfwcvtbf16_f_f_v_f32m2(v.reg, VEC_ELEM_NUM)) {};
-#else
- explicit FP32Vec8(const BF16Vec8& v) : reg(v.reg_fp32) {};
+#ifndef __riscv_v_min_vlen
+ #error "compiler did not define __riscv_v_min_vlen; pass -march=...zvlb"
#endif
- float reduce_sum() const {
- fixed_vfloat32m1_t scalar = __riscv_vfmv_s_f_f32m1(0.0f, 1);
- scalar = __riscv_vfredusum_vs_f32m2_f32m1(reg, scalar, VEC_ELEM_NUM);
- return __riscv_vfmv_f_s_f32m1_f32(scalar);
- }
-
- FP32Vec8 operator*(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfmul_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec8 operator+(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfadd_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec8 operator-(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfsub_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec8 operator/(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfdiv_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
-
- FP32Vec8 min(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfmin_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec8 max(const FP32Vec8& b) const {
- return FP32Vec8(__riscv_vfmax_vv_f32m2(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec8 abs() const {
- return FP32Vec8(__riscv_vfabs_v_f32m2(reg, VEC_ELEM_NUM));
- }
-
- FP32Vec8 min(const FP32Vec8& b, int elem_num) const {
- return FP32Vec8(__riscv_vfmin_vv_f32m2(reg, b.reg, elem_num));
- }
- FP32Vec8 max(const FP32Vec8& b, int elem_num) const {
- return FP32Vec8(__riscv_vfmax_vv_f32m2(reg, b.reg, elem_num));
- }
-
- FP32Vec8 clamp(const FP32Vec8& min_v, const FP32Vec8& max_v) const {
- fixed_vfloat32m2_t temp =
- __riscv_vfmax_vv_f32m2(min_v.reg, reg, VEC_ELEM_NUM);
- return FP32Vec8(__riscv_vfmin_vv_f32m2(max_v.reg, temp, VEC_ELEM_NUM));
- }
-
- void save(float* ptr) const { __riscv_vse32_v_f32m2(ptr, reg, VEC_ELEM_NUM); }
- void save(float* ptr, int elem_num) const {
- __riscv_vse32_v_f32m2(ptr, reg, elem_num);
- }
- void save_strided(float* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(float);
- __riscv_vsse32_v_f32m2(ptr, byte_stride, reg, VEC_ELEM_NUM);
- }
-
- FP32Vec8 exp() const {
- const float inv_ln2 = 1.44269504088896341f;
- fixed_vfloat32m2_t x_scaled =
- __riscv_vfmul_vf_f32m2(reg, inv_ln2, VEC_ELEM_NUM);
- fixed_vint32m2_t n_int = __riscv_vfcvt_x_f_v_i32m2(x_scaled, VEC_ELEM_NUM);
- fixed_vfloat32m2_t n_float = __riscv_vfcvt_f_x_v_f32m2(n_int, VEC_ELEM_NUM);
-
- fixed_vfloat32m2_t r =
- __riscv_vfsub_vv_f32m2(x_scaled, n_float, VEC_ELEM_NUM);
-
- fixed_vfloat32m2_t poly =
- __riscv_vfmv_v_f_f32m2(0.001333355810164f, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, r, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(poly, 0.009618129107628f, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, r, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(poly, 0.055504108664821f, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, r, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(poly, 0.240226506959101f, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, r, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(poly, 0.693147180559945f, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, r, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(poly, 1.0f, VEC_ELEM_NUM);
-
- fixed_vint32m2_t biased_exp =
- __riscv_vadd_vx_i32m2(n_int, 127, VEC_ELEM_NUM);
- biased_exp = __riscv_vmax_vx_i32m2(biased_exp, 0, VEC_ELEM_NUM);
- fixed_vint32m2_t exponent_bits =
- __riscv_vsll_vx_i32m2(biased_exp, 23, VEC_ELEM_NUM);
- fixed_vfloat32m2_t scale =
- __riscv_vreinterpret_v_i32m2_f32m2(exponent_bits);
-
- return FP32Vec8(__riscv_vfmul_vv_f32m2(poly, scale, VEC_ELEM_NUM));
- }
-
- FP32Vec8 tanh() const {
- fixed_vfloat32m2_t x_clamped = __riscv_vfmin_vf_f32m2(
- __riscv_vfmax_vf_f32m2(reg, -9.0f, VEC_ELEM_NUM), 9.0f, VEC_ELEM_NUM);
- fixed_vfloat32m2_t x2 =
- __riscv_vfmul_vf_f32m2(x_clamped, 2.0f, VEC_ELEM_NUM);
- FP32Vec8 exp_val = FP32Vec8(x2).exp();
- fixed_vfloat32m2_t num =
- __riscv_vfsub_vf_f32m2(exp_val.reg, 1.0f, VEC_ELEM_NUM);
- fixed_vfloat32m2_t den =
- __riscv_vfadd_vf_f32m2(exp_val.reg, 1.0f, VEC_ELEM_NUM);
- return FP32Vec8(__riscv_vfdiv_vv_f32m2(num, den, VEC_ELEM_NUM));
- }
-
- FP32Vec8 er() const {
- const float p = 0.3275911f, a1 = 0.254829592f, a2 = -0.284496736f,
- a3 = 1.421413741f, a4 = -1.453152027f, a5 = 1.061405429f;
- fixed_vfloat32m2_t abs_x = __riscv_vfabs_v_f32m2(reg, VEC_ELEM_NUM);
-
- fixed_vfloat32m2_t t = __riscv_vfadd_vf_f32m2(
- __riscv_vfmul_vf_f32m2(abs_x, p, VEC_ELEM_NUM), 1.0f, VEC_ELEM_NUM);
- t = __riscv_vfrdiv_vf_f32m2(t, 1.0f, VEC_ELEM_NUM);
-
- fixed_vfloat32m2_t poly = __riscv_vfmv_v_f_f32m2(a5, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(__riscv_vfmul_vv_f32m2(poly, t, VEC_ELEM_NUM),
- a4, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(__riscv_vfmul_vv_f32m2(poly, t, VEC_ELEM_NUM),
- a3, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(__riscv_vfmul_vv_f32m2(poly, t, VEC_ELEM_NUM),
- a2, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m2(__riscv_vfmul_vv_f32m2(poly, t, VEC_ELEM_NUM),
- a1, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m2(poly, t, VEC_ELEM_NUM);
-
- fixed_vfloat32m2_t exp_val =
- FP32Vec8(__riscv_vfneg_v_f32m2(
- __riscv_vfmul_vv_f32m2(abs_x, abs_x, VEC_ELEM_NUM),
- VEC_ELEM_NUM))
- .exp()
- .reg;
- fixed_vfloat32m2_t res = __riscv_vfrsub_vf_f32m2(
- __riscv_vfmul_vv_f32m2(poly, exp_val, VEC_ELEM_NUM), 1.0f,
- VEC_ELEM_NUM);
-
- vbool16_t mask = __riscv_vmflt_vf_f32m2_b16(reg, 0.0f, VEC_ELEM_NUM);
- return FP32Vec8(__riscv_vfneg_v_f32m2_m(mask, res, VEC_ELEM_NUM));
- }
-};
-
-struct FP32Vec16 : public Vec {
- constexpr static int VEC_ELEM_NUM = 16;
- fixed_vfloat32m4_t reg;
-
- explicit FP32Vec16(float v) : reg(__riscv_vfmv_v_f_f32m4(v, VEC_ELEM_NUM)) {};
- explicit FP32Vec16() : reg(__riscv_vfmv_v_f_f32m4(0.0f, VEC_ELEM_NUM)) {};
- explicit FP32Vec16(const float* ptr)
- : reg(__riscv_vle32_v_f32m4(ptr, VEC_ELEM_NUM)) {};
- explicit FP32Vec16(fixed_vfloat32m4_t data) : reg(data) {};
- explicit FP32Vec16(const FP32Vec8& data)
- : reg(__riscv_vcreate_v_f32m2_f32m4(data.reg, data.reg)) {};
- explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {};
- explicit FP32Vec16(const FP16Vec16& v);
-
-#ifdef RISCV_BF16_SUPPORT
- explicit FP32Vec16(fixed_vbfloat16m2_t v)
- : reg(__riscv_vfwcvtbf16_f_f_v_f32m4(v, VEC_ELEM_NUM)) {};
- explicit FP32Vec16(const BF16Vec16& v)
- : reg(__riscv_vfwcvtbf16_f_f_v_f32m4(v.reg, VEC_ELEM_NUM)) {};
-#else
- explicit FP32Vec16(const BF16Vec16& v) : reg(v.reg_fp32) {};
-#endif
-
- FP32Vec16 operator+(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfadd_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec16 operator-(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfsub_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec16 operator*(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfmul_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec16 operator/(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfdiv_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
-
- FP32Vec16 fma(const FP32Vec16& a, const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfmacc_vv_f32m4(reg, a.reg, b.reg, VEC_ELEM_NUM));
- }
-
- float reduce_sum() const {
- fixed_vfloat32m1_t scalar = __riscv_vfmv_s_f_f32m1(0.0f, 1);
- scalar = __riscv_vfredusum_vs_f32m4_f32m1(reg, scalar, VEC_ELEM_NUM);
- return __riscv_vfmv_f_s_f32m1_f32(scalar);
- }
-
- float reduce_max() const {
- fixed_vfloat32m1_t scalar =
- __riscv_vfmv_s_f_f32m1(std::numeric_limits::lowest(), 1);
- scalar = __riscv_vfredmax_vs_f32m4_f32m1(reg, scalar, VEC_ELEM_NUM);
- return __riscv_vfmv_f_s_f32m1_f32(scalar);
- }
-
- float reduce_min() const {
- fixed_vfloat32m1_t scalar =
- __riscv_vfmv_s_f_f32m1(std::numeric_limits::max(), 1);
- scalar = __riscv_vfredmin_vs_f32m4_f32m1(reg, scalar, VEC_ELEM_NUM);
- return __riscv_vfmv_f_s_f32m1_f32(scalar);
- }
-
- template
- float reduce_sub_sum(int idx) {
- static_assert(VEC_ELEM_NUM % group_size == 0);
- const int start = idx * group_size;
- vuint32m4_t indices = __riscv_vid_v_u32m4(VEC_ELEM_NUM);
- vbool8_t mask = __riscv_vmand_mm_b8(
- __riscv_vmsgeu_vx_u32m4_b8(indices, start, VEC_ELEM_NUM),
- __riscv_vmsltu_vx_u32m4_b8(indices, start + group_size, VEC_ELEM_NUM),
- VEC_ELEM_NUM);
- fixed_vfloat32m1_t scalar = __riscv_vfmv_s_f_f32m1(0.0f, 1);
- scalar =
- __riscv_vfredusum_vs_f32m4_f32m1_m(mask, reg, scalar, VEC_ELEM_NUM);
- return __riscv_vfmv_f_s_f32m1_f32(scalar);
- };
-
- FP32Vec16 max(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfmax_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec16 min(const FP32Vec16& b) const {
- return FP32Vec16(__riscv_vfmin_vv_f32m4(reg, b.reg, VEC_ELEM_NUM));
- }
- FP32Vec16 abs() const {
- return FP32Vec16(__riscv_vfabs_v_f32m4(reg, VEC_ELEM_NUM));
- }
-
- FP32Vec16 clamp(const FP32Vec16& min_v, const FP32Vec16& max_v) const {
- return FP32Vec16(__riscv_vfmin_vv_f32m4(
- max_v.reg, __riscv_vfmax_vv_f32m4(min_v.reg, reg, VEC_ELEM_NUM),
- VEC_ELEM_NUM));
- }
-
- void save(float* ptr) const { __riscv_vse32_v_f32m4(ptr, reg, VEC_ELEM_NUM); }
- void save(float* ptr, int elem_num) const {
- __riscv_vse32_v_f32m4(ptr, reg, elem_num);
- }
- void save_strided(float* ptr, ptrdiff_t stride) const {
- ptrdiff_t byte_stride = stride * sizeof(float);
- __riscv_vsse32_v_f32m4(ptr, byte_stride, reg, VEC_ELEM_NUM);
- }
-
- FP32Vec16 exp() const {
- const float inv_ln2 = 1.44269504088896341f;
- fixed_vfloat32m4_t x_scaled =
- __riscv_vfmul_vf_f32m4(reg, inv_ln2, VEC_ELEM_NUM);
- fixed_vint32m4_t n_int = __riscv_vfcvt_x_f_v_i32m4(x_scaled, VEC_ELEM_NUM);
- fixed_vfloat32m4_t n_float = __riscv_vfcvt_f_x_v_f32m4(n_int, VEC_ELEM_NUM);
- fixed_vfloat32m4_t r =
- __riscv_vfsub_vv_f32m4(x_scaled, n_float, VEC_ELEM_NUM);
-
- fixed_vfloat32m4_t poly =
- __riscv_vfmv_v_f_f32m4(0.001333355810164f, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, r, VEC_ELEM_NUM),
- 0.009618129107628f, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, r, VEC_ELEM_NUM),
- 0.055504108664821f, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, r, VEC_ELEM_NUM),
- 0.240226506959101f, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, r, VEC_ELEM_NUM),
- 0.693147180559945f, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, r, VEC_ELEM_NUM),
- 1.0f, VEC_ELEM_NUM);
-
- fixed_vint32m4_t biased_exp = __riscv_vmax_vx_i32m4(
- __riscv_vadd_vx_i32m4(n_int, 127, VEC_ELEM_NUM), 0, VEC_ELEM_NUM);
- fixed_vfloat32m4_t scale = __riscv_vreinterpret_v_i32m4_f32m4(
- __riscv_vsll_vx_i32m4(biased_exp, 23, VEC_ELEM_NUM));
-
- return FP32Vec16(__riscv_vfmul_vv_f32m4(poly, scale, VEC_ELEM_NUM));
- }
-
- FP32Vec16 tanh() const {
- fixed_vfloat32m4_t x_clamped = __riscv_vfmin_vf_f32m4(
- __riscv_vfmax_vf_f32m4(reg, -9.0f, VEC_ELEM_NUM), 9.0f, VEC_ELEM_NUM);
- FP32Vec16 exp_val =
- FP32Vec16(__riscv_vfmul_vf_f32m4(x_clamped, 2.0f, VEC_ELEM_NUM)).exp();
- return FP32Vec16(__riscv_vfdiv_vv_f32m4(
- __riscv_vfsub_vf_f32m4(exp_val.reg, 1.0f, VEC_ELEM_NUM),
- __riscv_vfadd_vf_f32m4(exp_val.reg, 1.0f, VEC_ELEM_NUM), VEC_ELEM_NUM));
- }
-
- FP32Vec16 er() const {
- const float p = 0.3275911f, a1 = 0.254829592f, a2 = -0.284496736f,
- a3 = 1.421413741f, a4 = -1.453152027f, a5 = 1.061405429f;
- fixed_vfloat32m4_t abs_x = __riscv_vfabs_v_f32m4(reg, VEC_ELEM_NUM);
- fixed_vfloat32m4_t t = __riscv_vfrdiv_vf_f32m4(
- __riscv_vfadd_vf_f32m4(__riscv_vfmul_vf_f32m4(abs_x, p, VEC_ELEM_NUM),
- 1.0f, VEC_ELEM_NUM),
- 1.0f, VEC_ELEM_NUM);
-
- fixed_vfloat32m4_t poly = __riscv_vfmv_v_f_f32m4(a5, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, t, VEC_ELEM_NUM),
- a4, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, t, VEC_ELEM_NUM),
- a3, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, t, VEC_ELEM_NUM),
- a2, VEC_ELEM_NUM);
- poly = __riscv_vfadd_vf_f32m4(__riscv_vfmul_vv_f32m4(poly, t, VEC_ELEM_NUM),
- a1, VEC_ELEM_NUM);
- poly = __riscv_vfmul_vv_f32m4(poly, t, VEC_ELEM_NUM);
-
- fixed_vfloat32m4_t exp_val =
- FP32Vec16(__riscv_vfneg_v_f32m4(
- __riscv_vfmul_vv_f32m4(abs_x, abs_x, VEC_ELEM_NUM),
- VEC_ELEM_NUM))
- .exp()
- .reg;
- fixed_vfloat32m4_t res = __riscv_vfrsub_vf_f32m4(
- __riscv_vfmul_vv_f32m4(poly, exp_val, VEC_ELEM_NUM), 1.0f,
- VEC_ELEM_NUM);
-
- vbool8_t mask = __riscv_vmflt_vf_f32m4_b8(reg, 0.0f, VEC_ELEM_NUM);
- return FP32Vec16(__riscv_vfneg_v_f32m4_m(mask, res, VEC_ELEM_NUM));
- }
-};
-
-// ============================================================================
-// Type Traits & Global Helpers
-// ============================================================================
-
-template
-struct VecType {
- using vec_type = void;
- using vec_t = void;
-};
-
-template
-using vec_t = typename VecType::vec_type;
-
-template <>
-struct VecType {
- using vec_type = FP32Vec8;
- using vec_t = FP32Vec8;
-};
-template <>
-struct VecType {
- using vec_type = FP16Vec8;
- using vec_t = FP16Vec8;
-};
-template <>
-struct VecType {
- using vec_type = BF16Vec8;
- using vec_t = BF16Vec8;
-};
-
-template
-void storeFP32(float v, T* ptr) {
- *ptr = v;
-}
-template <>
-inline void storeFP32(float v, c10::Half* ptr) {
- *reinterpret_cast<_Float16*>(ptr) = static_cast<_Float16>(v);
-}
-
-inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
- reg = __riscv_vfncvt_f_f_w_f16m2(v.reg, VEC_ELEM_NUM);
-}
-inline FP16Vec8::FP16Vec8(const FP32Vec8& v) {
- reg = __riscv_vfncvt_f_f_w_f16m1(v.reg, VEC_ELEM_NUM);
-}
-inline FP32Vec16::FP32Vec16(const FP16Vec16& v) {
- reg = __riscv_vfwcvt_f_f_v_f32m4(v.reg, VEC_ELEM_NUM);
-}
-inline void fma(FP32Vec16& acc, const FP32Vec16& a, const FP32Vec16& b) {
- acc = acc.fma(a, b);
-}
-
-#ifdef RISCV_BF16_SUPPORT
-template <>
-inline void storeFP32(float v, c10::BFloat16* ptr) {
- *ptr = static_cast<__bf16>(v);
-};
-inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
- : reg(__riscv_vfncvtbf16_f_f_w_bf16m1(v.reg, VEC_ELEM_NUM)) {};
-inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
- : reg(__riscv_vfncvtbf16_f_f_w_bf16m2(v.reg, VEC_ELEM_NUM)) {};
-#else
-template <>
-inline void storeFP32(float v, c10::BFloat16* ptr) {
- uint32_t val;
- std::memcpy(&val, &v, 4);
- *reinterpret_cast(ptr) = static_cast(val >> 16);
-}
-inline BF16Vec8::BF16Vec8(const FP32Vec8& v) : reg_fp32(v.reg) {}
-inline BF16Vec16::BF16Vec16(const FP32Vec16& v) : reg_fp32(v.reg) {}
-#endif
-
-inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); }
-
-} // namespace vec_op
-
-#ifndef CPU_KERNEL_GUARD_IN
- #define CPU_KERNEL_GUARD_IN(NAME)
-#endif
-
-#ifndef CPU_KERNEL_GUARD_OUT
- #define CPU_KERNEL_GUARD_OUT(NAME)
-#endif
+#include "cpu_types_riscv_defs.hpp"
+#include "cpu_types_riscv_impl.hpp"
-#endif // CPU_TYPES_RISCV_HPP
\ No newline at end of file
+#endif // CPU_TYPES_RISCV_HPP
diff --git a/csrc/cpu/cpu_types_riscv_defs.hpp b/csrc/cpu/cpu_types_riscv_defs.hpp
new file mode 100644
index 000000000000..04ce2728f032
--- /dev/null
+++ b/csrc/cpu/cpu_types_riscv_defs.hpp
@@ -0,0 +1,98 @@
+#ifndef CPU_TYPES_RISCV_DEFS_HPP
+#define CPU_TYPES_RISCV_DEFS_HPP
+
+// VLEN-to-LMUL mapping for RISC-V Vector extension.
+//
+// LMUL_ expands to the LMUL suffix giving N total bits of vector data:
+// VLEN=128: LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8
+// VLEN=256: LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4
+
+#include
+
+#if __riscv_v_min_vlen == 128
+ #define LMUL_128 m1
+ #define LMUL_256 m2
+ #define LMUL_512 m4
+ #define LMUL_1024 m8
+ #define BOOL_256 b16
+ #define BOOL_512 b8
+#elif __riscv_v_min_vlen == 256
+ #define LMUL_128 mf2
+ #define LMUL_256 m1
+ #define LMUL_512 m2
+ #define LMUL_1024 m4
+ #define BOOL_256 b32
+ #define BOOL_512 b16
+#else
+ #error "cpu_types_riscv_defs.hpp: unsupported __riscv_v_min_vlen"
+#endif
+
+// Token-paste helpers.
+#define _RVV_P2(a, b) a##b
+#define _RVV_P3(a, b, c) a##b##c
+#define _RVV_P4(a, b, c, d) a##b##c##d
+#define RVVTYPE(base, lmul, suffix) _RVV_P3(base, lmul, suffix)
+#define RVVI(base, lmul) _RVV_P2(base, lmul)
+#define RVVI3(base, lmul, suffix) _RVV_P3(base, lmul, suffix)
+#define RVVI4(a, b, c, d) _RVV_P4(a, b, c, d)
+// For mask intrinsics: RVVIB(base, LMUL_256, BOOL_256) → base##m2##_##b16
+#define _RVV_PB(base, lmul, btype) base##lmul##_##btype
+#define RVVIB(base, lmul, btype) _RVV_PB(base, lmul, btype)
+
+// ---- Semantic fixed-vector typedefs (named by element count) ----
+
+// float16
+typedef RVVTYPE(vfloat16, LMUL_128, _t) fixed_fp16x8_t
+ __attribute__((riscv_rvv_vector_bits(128)));
+typedef RVVTYPE(vfloat16, LMUL_256, _t) fixed_fp16x16_t
+ __attribute__((riscv_rvv_vector_bits(256)));
+
+// float32
+typedef RVVTYPE(vfloat32, LMUL_128, _t) fixed_fp32x4_t
+ __attribute__((riscv_rvv_vector_bits(128)));
+typedef RVVTYPE(vfloat32, LMUL_256, _t) fixed_fp32x8_t
+ __attribute__((riscv_rvv_vector_bits(256)));
+typedef RVVTYPE(vfloat32, LMUL_512, _t) fixed_fp32x16_t
+ __attribute__((riscv_rvv_vector_bits(512)));
+typedef RVVTYPE(vfloat32, LMUL_1024, _t) fixed_fp32x32_t
+ __attribute__((riscv_rvv_vector_bits(1024)));
+
+// int32
+typedef RVVTYPE(vint32, LMUL_256, _t) fixed_i32x8_t
+ __attribute__((riscv_rvv_vector_bits(256)));
+typedef RVVTYPE(vint32, LMUL_512, _t) fixed_i32x16_t
+ __attribute__((riscv_rvv_vector_bits(512)));
+
+// uint16
+typedef RVVTYPE(vuint16, LMUL_128, _t) fixed_u16x8_t
+ __attribute__((riscv_rvv_vector_bits(128)));
+typedef RVVTYPE(vuint16, LMUL_256, _t) fixed_u16x16_t
+ __attribute__((riscv_rvv_vector_bits(256)));
+typedef RVVTYPE(vuint16, LMUL_512, _t) fixed_u16x32_t
+ __attribute__((riscv_rvv_vector_bits(512)));
+
+// bfloat16
+#ifdef RISCV_BF16_SUPPORT
+typedef RVVTYPE(vbfloat16, LMUL_128, _t) fixed_bf16x8_t
+ __attribute__((riscv_rvv_vector_bits(128)));
+typedef RVVTYPE(vbfloat16, LMUL_256, _t) fixed_bf16x16_t
+ __attribute__((riscv_rvv_vector_bits(256)));
+typedef RVVTYPE(vbfloat16, LMUL_512, _t) fixed_bf16x32_t
+ __attribute__((riscv_rvv_vector_bits(512)));
+#endif
+
+// ---- Reduction accumulator type (always m1 = one register of f32) ----
+// Used for scalar reductions; only element [0] is meaningful.
+typedef vfloat32m1_t rvv_f32_accum_t
+ __attribute__((riscv_rvv_vector_bits(__riscv_v_min_vlen)));
+
+// ---- Mask types for f32 elements ----
+#if __riscv_v_min_vlen == 128
+typedef vbool16_t rvv_mask_f32x8_t;
+typedef vbool8_t rvv_mask_f32x16_t;
+#elif __riscv_v_min_vlen == 256
+typedef vbool32_t rvv_mask_f32x8_t;
+typedef vbool16_t rvv_mask_f32x16_t;
+#endif
+
+#endif // CPU_TYPES_RISCV_DEFS_HPP
diff --git a/csrc/cpu/cpu_types_riscv_impl.hpp b/csrc/cpu/cpu_types_riscv_impl.hpp
new file mode 100644
index 000000000000..3952a811c2c5
--- /dev/null
+++ b/csrc/cpu/cpu_types_riscv_impl.hpp
@@ -0,0 +1,905 @@
+#ifndef CPU_TYPES_RISCV_IMPL_HPP
+#define CPU_TYPES_RISCV_IMPL_HPP
+
+// Shared implementation of RVV vector-type wrapper classes.
+// This file is VLEN-independent: it uses the semantic type names and
+// RVVI() intrinsic macros from cpu_types_riscv_defs.hpp.
+//
+// DO NOT include this file directly; include cpu_types_riscv.hpp instead.
+
+#include
+#include
+#include
+#include
+#include
+#include
+namespace vec_op {
+
+// BFloat16 is always supported on RISC-V: natively when RISCV_BF16_SUPPORT
+// is defined, otherwise via the FP32-simulation fallback path.
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
+
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
+
+#define FORCE_INLINE __attribute__((always_inline)) inline
+
+namespace {
+template
+constexpr void unroll_loop_item(std::integer_sequence, F&& f) {
+ (f(std::integral_constant{}), ...);
+};
+} // namespace
+
+template >>
+constexpr void unroll_loop(F&& f) {
+ unroll_loop_item(std::make_integer_sequence{}, std::forward(f));
+}
+
+template
+struct Vec {
+ constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; };
+};
+
+struct FP32Vec8;
+struct FP32Vec16;
+
+// ============================================================================
+// FP16 Implementation
+// ============================================================================
+
+struct FP16Vec8 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 8;
+ fixed_fp16x8_t reg;
+
+ explicit FP16Vec8(const void* ptr)
+ : reg(RVVI(__riscv_vle16_v_f16, LMUL_128)(
+ static_cast(ptr), VEC_ELEM_NUM)) {};
+
+ explicit FP16Vec8(const FP32Vec8&);
+
+ void save(void* ptr) const {
+ RVVI(__riscv_vse16_v_f16, LMUL_128)(static_cast<_Float16*>(ptr), reg,
+ VEC_ELEM_NUM);
+ }
+ void save(void* ptr, int elem_num) const {
+ RVVI(__riscv_vse16_v_f16, LMUL_128)(static_cast<_Float16*>(ptr), reg,
+ elem_num);
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ ptrdiff_t byte_stride = stride * sizeof(_Float16);
+ RVVI(__riscv_vsse16_v_f16, LMUL_128)(static_cast<_Float16*>(ptr),
+ byte_stride, reg, VEC_ELEM_NUM);
+ }
+};
+
+struct FP16Vec16 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 16;
+ fixed_fp16x16_t reg;
+
+ explicit FP16Vec16(const void* ptr)
+ : reg(RVVI(__riscv_vle16_v_f16, LMUL_256)(
+ static_cast(ptr), VEC_ELEM_NUM)) {};
+
+ explicit FP16Vec16(const FP32Vec16& vec);
+
+ void save(void* ptr) const {
+ RVVI(__riscv_vse16_v_f16, LMUL_256)(static_cast<_Float16*>(ptr), reg,
+ VEC_ELEM_NUM);
+ }
+ void save(void* ptr, int elem_num) const {
+ RVVI(__riscv_vse16_v_f16, LMUL_256)(static_cast<_Float16*>(ptr), reg,
+ elem_num);
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ ptrdiff_t byte_stride = stride * sizeof(_Float16);
+ RVVI(__riscv_vsse16_v_f16, LMUL_256)(static_cast<_Float16*>(ptr),
+ byte_stride, reg, VEC_ELEM_NUM);
+ }
+};
+
+// ============================================================================
+// BF16 Implementation
+// ============================================================================
+
+#ifdef RISCV_BF16_SUPPORT
+
+FORCE_INLINE fixed_u16x8_t bf16_to_u16(fixed_bf16x8_t v) {
+ return RVVI4(__riscv_vreinterpret_v_bf16, LMUL_128, _u16, LMUL_128)(v);
+}
+FORCE_INLINE fixed_u16x16_t bf16_to_u16(fixed_bf16x16_t v) {
+ return RVVI4(__riscv_vreinterpret_v_bf16, LMUL_256, _u16, LMUL_256)(v);
+}
+FORCE_INLINE fixed_u16x32_t bf16_to_u16(fixed_bf16x32_t v) {
+ return RVVI4(__riscv_vreinterpret_v_bf16, LMUL_512, _u16, LMUL_512)(v);
+}
+
+struct BF16Vec8 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 8;
+ fixed_bf16x8_t reg;
+
+ explicit BF16Vec8(const void* ptr)
+ : reg(RVVI4(__riscv_vreinterpret_v_u16, LMUL_128, _bf16,
+ LMUL_128)(RVVI(__riscv_vle16_v_u16, LMUL_128)(
+ reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
+
+ explicit BF16Vec8(fixed_bf16x8_t data) : reg(data) {};
+ explicit BF16Vec8(const FP32Vec8&);
+
+ void save(void* ptr) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_128)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), VEC_ELEM_NUM);
+ }
+ void save(void* ptr, int elem_num) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_128)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), elem_num);
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ ptrdiff_t byte_stride = stride * sizeof(uint16_t);
+ RVVI(__riscv_vsse16_v_u16, LMUL_128)(reinterpret_cast(ptr),
+ byte_stride, bf16_to_u16(reg),
+ VEC_ELEM_NUM);
+ }
+};
+
+struct BF16Vec16 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 16;
+ fixed_bf16x16_t reg;
+
+ explicit BF16Vec16(const void* ptr)
+ : reg(RVVI4(__riscv_vreinterpret_v_u16, LMUL_256, _bf16,
+ LMUL_256)(RVVI(__riscv_vle16_v_u16, LMUL_256)(
+ reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
+
+ explicit BF16Vec16(fixed_bf16x16_t data) : reg(data) {};
+ explicit BF16Vec16(const FP32Vec16&);
+
+ void save(void* ptr) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_256)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), VEC_ELEM_NUM);
+ }
+ void save(void* ptr, int elem_num) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_256)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), elem_num);
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ ptrdiff_t byte_stride = stride * sizeof(uint16_t);
+ RVVI(__riscv_vsse16_v_u16, LMUL_256)(reinterpret_cast(ptr),
+ byte_stride, bf16_to_u16(reg),
+ VEC_ELEM_NUM);
+ }
+};
+
+struct BF16Vec32 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 32;
+ fixed_bf16x32_t reg;
+
+ explicit BF16Vec32(const void* ptr)
+ : reg(RVVI4(__riscv_vreinterpret_v_u16, LMUL_512, _bf16,
+ LMUL_512)(RVVI(__riscv_vle16_v_u16, LMUL_512)(
+ reinterpret_cast(ptr), VEC_ELEM_NUM))) {};
+
+ explicit BF16Vec32(fixed_bf16x32_t data) : reg(data) {};
+
+ explicit BF16Vec32(const BF16Vec8& v) {
+ fixed_u16x8_t u16_val = bf16_to_u16(v.reg);
+ fixed_u16x32_t u16_combined =
+ RVVI4(__riscv_vcreate_v_u16, LMUL_128, _u16, LMUL_512)(
+ u16_val, u16_val, u16_val, u16_val);
+ reg = RVVI4(__riscv_vreinterpret_v_u16, LMUL_512, _bf16,
+ LMUL_512)(u16_combined);
+ };
+
+ void save(void* ptr) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_512)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), VEC_ELEM_NUM);
+ }
+ void save(void* ptr, int elem_num) const {
+ RVVI(__riscv_vse16_v_u16, LMUL_512)(reinterpret_cast(ptr),
+ bf16_to_u16(reg), elem_num);
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ ptrdiff_t byte_stride = stride * sizeof(uint16_t);
+ RVVI(__riscv_vsse16_v_u16, LMUL_512)(reinterpret_cast(ptr),
+ byte_stride, bf16_to_u16(reg),
+ VEC_ELEM_NUM);
+ }
+};
+
+#else
+// ============================================================================
+// BF16 Fallback Implementation (FP32 Simulation)
+// ============================================================================
+
+struct BF16Vec8 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 8;
+ fixed_fp32x8_t reg_fp32;
+ explicit BF16Vec8(const void* ptr) {
+ const uint16_t* u16 = static_cast(ptr);
+ float tmp[8];
+ for (int i = 0; i < 8; ++i) {
+ uint32_t v = static_cast(u16[i]) << 16;
+ std::memcpy(&tmp[i], &v, 4);
+ }
+ reg_fp32 = RVVI(__riscv_vle32_v_f32, LMUL_256)(tmp, 8);
+ }
+ explicit BF16Vec8(const FP32Vec8&);
+ void save(void* ptr) const {
+ float tmp[8];
+ RVVI(__riscv_vse32_v_f32, LMUL_256)(tmp, reg_fp32, 8);
+ uint16_t* u16 = static_cast(ptr);
+ for (int i = 0; i < 8; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ u16[i] = static_cast(v >> 16);
+ }
+ }
+ void save(void* ptr, int elem_num) const {
+ float tmp[8];
+ RVVI(__riscv_vse32_v_f32, LMUL_256)(tmp, reg_fp32, 8);
+ uint16_t* u16 = static_cast(ptr);
+ for (int i = 0; i < elem_num; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ u16[i] = static_cast(v >> 16);
+ }
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ float tmp[8];
+ RVVI(__riscv_vse32_v_f32, LMUL_256)(tmp, reg_fp32, 8);
+ uint8_t* u8 = static_cast(ptr);
+ ptrdiff_t byte_stride = stride * sizeof(uint16_t);
+ for (int i = 0; i < 8; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ uint16_t val = static_cast(v >> 16);
+ *reinterpret_cast(u8 + i * byte_stride) = val;
+ }
+ }
+};
+
+struct BF16Vec16 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 16;
+ fixed_fp32x16_t reg_fp32;
+ explicit BF16Vec16(const void* ptr) {
+ const uint16_t* u16 = static_cast(ptr);
+ float tmp[16];
+ for (int i = 0; i < 16; ++i) {
+ uint32_t v = static_cast(u16[i]) << 16;
+ std::memcpy(&tmp[i], &v, 4);
+ }
+ reg_fp32 = RVVI(__riscv_vle32_v_f32, LMUL_512)(tmp, 16);
+ }
+ explicit BF16Vec16(const FP32Vec16&);
+ void save(void* ptr) const {
+ float tmp[16];
+ RVVI(__riscv_vse32_v_f32, LMUL_512)(tmp, reg_fp32, 16);
+ uint16_t* u16 = static_cast(ptr);
+ for (int i = 0; i < 16; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ u16[i] = static_cast(v >> 16);
+ }
+ }
+ void save(void* ptr, int elem_num) const {
+ float tmp[16];
+ RVVI(__riscv_vse32_v_f32, LMUL_512)(tmp, reg_fp32, 16);
+ uint16_t* u16 = static_cast(ptr);
+ for (int i = 0; i < elem_num; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ u16[i] = static_cast(v >> 16);
+ }
+ }
+ void save_strided(void* ptr, ptrdiff_t stride) const {
+ float tmp[16];
+ RVVI(__riscv_vse32_v_f32, LMUL_512)(tmp, reg_fp32, 16);
+ uint8_t* u8 = static_cast(ptr);
+ ptrdiff_t byte_stride = stride * sizeof(uint16_t);
+ for (int i = 0; i < 16; ++i) {
+ uint32_t v;
+ std::memcpy(&v, &tmp[i], 4);
+ uint16_t val = static_cast(v >> 16);
+ *reinterpret_cast(u8 + i * byte_stride) = val;
+ }
+ }
+};
+
+struct BF16Vec32 : public Vec {
+ constexpr static int VEC_ELEM_NUM = 32;
+ fixed_fp32x32_t reg_fp32;
+
+ explicit BF16Vec32(const void* ptr) {
+ const uint16_t* u16 = static_cast(ptr);
+ float tmp[32];
+ for (int i = 0; i < 32; ++i) {
+ uint32_t v = static_cast(u16[i]) << 16;
+ std::memcpy(&tmp[i], &v, 4);
+ }
+ reg_fp32 = RVVI(__riscv_vle32_v_f32, LMUL_1024)(tmp, 32);
+ }
+
+ explicit BF16Vec32(const BF16Vec8& v) {
+ float tmp_small[8];
+ RVVI(__riscv_vse32_v_f32, LMUL_256)(tmp_small, v.reg_fp32, 8);
+ float tmp_large[32];
+ for (int i = 0; i < 4; ++i) {
+ std::memcpy(tmp_large + (i * 8), tmp_small, 8 * sizeof(float));
+ }
+ reg_fp32 = RVVI(__riscv_vle32_v_f32, LMUL_1024)(tmp_large, 32);
+ }
+
+ void save(void* ptr) const {
+ float tmp[32];
+ RVVI(__riscv_vse32_v_f32, LMUL_1024)(tmp, reg_fp32, 32);
+ uint16_t* u16 = static_cast