Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ def sparse_mqa_fwd(

is_hip = getattr(torch.version, "hip", None)
if is_hip:
# Limit pipeline buffering for 64-head HIP tiles to reduce LDS use.
kernel_num_stages = min(num_stages, 1) if H_per_block == 64 else num_stages
# Limit pipeline buffering for 32- and 64-head HIP tiles to reduce LDS use.
kernel_num_stages = min(num_stages, 1) if H_per_block >= 32 else num_stages
else:
kernel_num_stages = num_stages

Expand Down
9 changes: 6 additions & 3 deletions scripts/amd/run_deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class ScriptArgs(command_utils.ExecuteTrainConfig):
enable_eval: bool = True
enable_mtp: bool = False
dsv4_impl: Literal["miles"] = "miles"
cp_size: int | None = None

hf_checkpoint: str | None = None
data_dir: str = "/root/datasets"
Expand Down Expand Up @@ -219,7 +220,7 @@ def _prepare_spmd(args: ScriptArgs):

num_gpus_for_convert = actor_num_gpus_per_node
if is_4layer:
num_gpus_for_convert = min(num_gpus_for_convert, 4)
num_gpus_for_convert = 1

U.convert_checkpoint(
model_name=args.model_name,
Expand Down Expand Up @@ -265,11 +266,13 @@ def _get_parallel_config(args: ScriptArgs) -> str:

# Single-node smoke-test configs
if actor_num_nodes == 1:
cp_size = args.cp_size or 1
return (
f"--tensor-model-parallel-size {actor_num_gpus_per_node} "
f"--tensor-model-parallel-size {actor_num_gpus_per_node // cp_size} "
"--sequence-parallel "
"--pipeline-model-parallel-size 1 "
"--context-parallel-size 1 "
f"--context-parallel-size {cp_size} "
f"{'--allgather-cp ' if cp_size > 1 else ''}"
f"--expert-model-parallel-size {actor_num_gpus_per_node} "
"--expert-tensor-parallel-size 1 "
)
Expand Down
3 changes: 2 additions & 1 deletion tests/e2e/ckpt/test_qwen3_5_0.8B_ckpt.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci

from miles.utils.external_utils import command_utils

register_cuda_ci(est_time=1400, suite="stage-c-2-gpu-h200", labels=["ckpt"], hardware=["hopper", "blackwell"])
register_rocm_ci(est_time=1000, suite="nightly-stage-c-2-gpu-mi350", labels=["ckpt"])

ENABLE_EVAL = bool(int(os.environ.get("MILES_TEST_ENABLE_EVAL", "1")))

Expand Down
3 changes: 2 additions & 1 deletion tests/e2e/fsdp/test_qwen3_0.6B_fsdp_disk_delta.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci

import miles.utils.external_utils.command_utils.legacy as U

Expand All @@ -10,6 +10,7 @@
labels=["weight-update"],
hardware=["hopper"],
)
register_rocm_ci(est_time=500, suite="nightly-stage-c-4-gpu-mi350", labels=["weight-update"])

# The FSDP backend streams through the shared WeightUpdater, so every transfer
# protocol is open to it. disk-delta is the one that reconciles the stream
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@
import dataclasses
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.model_scripts import test_deepseek_v4_flash_4layer_ci as base

register_cuda_ci(
est_time=1900, suite="stage-c-4-gpu-h200", labels=["megatron", "model-scripts"], hardware=["hopper", "blackwell"]
)
register_rocm_ci(est_time=700, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "model-scripts"])

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os

from scripts.run_deepseek_v4 import ScriptArgs, _prepare_download, _prepare_single, _prepare_spmd, _train
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate

register_cuda_ci(
Expand All @@ -10,6 +10,11 @@
labels=["megatron", "model-scripts"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(
est_time=900,
suite="nightly-stage-c-4-gpu-mi350",
labels=["megatron", "model-scripts"],
)

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand All @@ -19,7 +24,7 @@


def _args() -> ScriptArgs:
return ScriptArgs(
args = ScriptArgs(
model_name="DeepSeek-V4-Flash-FP8-4layer",
task="gsm8k",
enable_eval=False,
Expand All @@ -34,6 +39,10 @@ def _args() -> ScriptArgs:
"--ci-test " "--check-weight-update-allow-quant-error " "--ci-disable-logprobs-checker " "--num-rollout 2 "
),
)
if os.getenv("MILES_HARDWARE_PLATFORM") == "rocm":
args.dsa_kernel_backend = "none"
args.extra_args += """--train-env-vars '{"NVTE_FP8_BLOCK_SCALING_FP32_SCALES":"0"}' """
return args


def prepare(args: ScriptArgs):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os

from scripts.run_inkling import _MODEL_REGISTRY, ScriptArgs, _train
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate


Expand All @@ -16,6 +16,11 @@
labels=["megatron", "model-scripts", "lora"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(
est_time=500,
suite="nightly-stage-c-4-gpu-mi350",
labels=["megatron", "model-scripts", "lora"],
)

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down Expand Up @@ -44,7 +49,8 @@ def _args() -> ScriptArgs:
"--check-weight-update-skip-list visual. audio. ._w1_delta ._a_cat "
"--ci-disable-logprobs-checker "
"--check-lora-weight-equal "
),
)
+ ("--sglang-attention-backend triton " if os.getenv("MILES_HARDWARE_PLATFORM") == "rocm" else ""),
)


Expand Down
8 changes: 7 additions & 1 deletion tests/e2e/megatron/model_scripts/test_kimi_k3_4layer_ci.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os

from scripts.run_kimi_k3 import ScriptArgs, _prepare_bf16, _prepare_download, _prepare_torch_dist, _train
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate

register_cuda_ci(
Expand All @@ -10,6 +10,11 @@
labels=["megatron", "model-scripts"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(
est_time=1100,
suite="nightly-stage-c-8-gpu-mi350",
labels=["megatron", "model-scripts"],
)

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down Expand Up @@ -38,6 +43,7 @@ def _args() -> ScriptArgs:
rollout_max_concurrency=16,
check_weight_update_equal=True,
skip_saving=True,
rollout_bf16=os.getenv("MILES_HARDWARE_PLATFORM") == "rocm",
extra_args="--ci-test --ci-disable-logprobs-checker ",
)

Expand Down
12 changes: 11 additions & 1 deletion tests/e2e/megatron/model_scripts/test_qwen3_8_next_4layer_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from scripts.run_qwen3_8_next import _MODEL_REGISTRY, ScriptArgs, _train

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate


Expand All @@ -12,6 +12,11 @@
labels=["megatron", "model-scripts"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(
est_time=600,
suite="nightly-stage-c-8-gpu-mi350",
labels=["megatron", "model-scripts"],
)

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand All @@ -33,6 +38,11 @@ def _args() -> ScriptArgs:
skip_saving=True,
extra_args=(
"--ci-test " "--ci-disable-kl-checker " "--ci-disable-logprobs-checker " "--offload-train-target cpu "
)
+ (
"--linear-attention-backend fla --sglang-linear-attn-prefill-backend triton "
if os.getenv("MILES_HARDWARE_PLATFORM") == "rocm"
else ""
),
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.test_glm47_flash._common import CaseConfig, execute, prepare

register_cuda_ci(est_time=1200, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"])
register_rocm_ci(est_time=1500, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"])

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@

import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.test_qwen3_30B_A3B._common import CaseConfig, execute, prepare

register_cuda_ci(est_time=1200, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"])
register_rocm_ci(est_time=1000, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"])

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@

import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.test_qwen3_5_35B_A3B._common import CaseConfig, execute, prepare

register_cuda_ci(
est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35"], hardware=["hopper", "blackwell"]
)
register_rocm_ci(est_time=1300, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"])

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@
no DeepEP on either side. Weights reach the engine by broadcast.
"""

import dataclasses
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.test_qwen3_5_35B_A3B._common import CaseConfig, execute, prepare

Expand All @@ -19,6 +20,11 @@
labels=["megatron", "qwen35", "weight-update", "fully-async", "replay"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(
est_time=1400,
suite="nightly-stage-c-8-gpu-mi350",
labels=["megatron", "qwen35", "weight-update", "fully-async", "replay"],
)

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down Expand Up @@ -56,4 +62,9 @@
for proxy_var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
os.environ.pop(proxy_var, None)
prepare(CASE)
if os.getenv("MILES_HARDWARE_PLATFORM") == "rocm":
CASE = dataclasses.replace(
CASE,
extra_args=CASE.extra_args + "--sglang-disable-shared-experts-fusion --debug-unified-grad-fused-logprob ",
)
execute(CASE, wandb_file=__file__)
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@
VLM/vision implementation on the training side, so they are never synced.
"""

import dataclasses
import os

from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.megatron.test_qwen3_5_35B_A3B._common import CaseConfig, execute, prepare

# Hopper only until a B200 run of this case passes: on Blackwell, FP8 without DeepEP pairs UE8M0
# linear scales with fp32 expert scales (quantizer_fp8.py).
register_cuda_ci(est_time=2100, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35"], hardware=["hopper"])
register_rocm_ci(est_time=1600, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"])

register_ci_gate(metric_key="train/grad_norm")
register_ci_gate(metric_key="train/ppo_kl")
Expand Down Expand Up @@ -57,4 +59,6 @@
for proxy_var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
os.environ.pop(proxy_var, None)
prepare(CASE)
if os.getenv("MILES_HARDWARE_PLATFORM") == "rocm":
CASE = dataclasses.replace(CASE, megatron_dispatcher="alltoall")
execute(CASE, wandb_file=__file__)
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.sglang.test_session_server_multi_role._common import ModelConfig, run_both_versions

register_cuda_ci(est_time=2100, suite="stage-c-4-gpu-h200", labels=["sglang"], hardware=["hopper", "blackwell"])
register_rocm_ci(est_time=3000, suite="nightly-stage-c-4-gpu-mi350", labels=["sglang"])
register_ci_gate(metric_key="rollout/tito_session_mismatch_rate/v1/assistant_text")
register_ci_gate(metric_key="rollout/tito_session_mismatch_rate/v2/assistant_text")

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from tests.ci.metric_history import register_ci_gate
from tests.e2e.sglang.test_session_server_multi_role._common import ModelConfig, run_both_versions

register_cuda_ci(est_time=900, suite="stage-c-2-gpu-h200", labels=["sglang"], hardware=["hopper", "blackwell"])
register_rocm_ci(est_time=600, suite="nightly-stage-c-2-gpu-mi350", labels=["sglang"])
register_ci_gate(metric_key="rollout/tito_session_mismatch_rate/v1/assistant_text")
register_ci_gate(metric_key="rollout/tito_session_mismatch_rate/v2/assistant_text")

Expand Down
3 changes: 2 additions & 1 deletion tests/fast-gpu/test_dsv4_indexer_cp_balance.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import torch
import torch.distributed as dist
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci

from miles_plugins.models.deepseek_v4.ops.cp_row_balance import RowExchange
from miles_plugins.models.deepseek_v4.ops.kernel.tilelang_indexer_fwd import (
Expand All @@ -36,6 +36,7 @@
labels=["precision", "megatron"],
hardware=["hopper", "blackwell"],
)
register_rocm_ci(est_time=60, suite="nightly-stage-c-4-gpu-mi350", labels=["precision", "megatron"])

SEQLEN_GLOBAL = 16384
RATIO = 4
Expand Down
3 changes: 2 additions & 1 deletion tests/fast-gpu/test_torch_native_offload.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import torch
import torch.distributed as dist
import torch.nn as nn
from tests.ci.ci_register import register_cuda_ci
from tests.ci.ci_register import register_cuda_ci, register_rocm_ci
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
from torch.distributed.tensor import DTensor

from miles.backends.training_utils.torch_native.offload import move_train_state

register_cuda_ci(est_time=30, suite="stage-b-2-gpu-h200", labels=["fsdp", "torchtitan"], hardware=["hopper"])
register_rocm_ci(est_time=60, suite="nightly-stage-c-2-gpu-mi350", labels=["fsdp", "torchtitan"])


def _local(tensor: torch.Tensor) -> torch.Tensor:
Expand Down
Loading