Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
eda7a2f
Merge 0227 (#11)
inkcherry Mar 2, 2026
d5a747c
refine code
inkcherry Mar 2, 2026
ab68bfc
fix refine
inkcherry Mar 2, 2026
27da046
update comment
inkcherry Mar 2, 2026
a11325a
update deferred out config
inkcherry Mar 2, 2026
72fc4bb
update proxy
inkcherry Mar 2, 2026
30a2f2d
delte scripst
inkcherry Mar 2, 2026
4e10fe8
clean up
inkcherry Mar 2, 2026
a076f00
rename
inkcherry Mar 2, 2026
d1579bc
Merge off/main into atom_pd
inkcherry Mar 19, 2026
2e138b1
fix merge
inkcherry Mar 19, 2026
b625f27
fix non-pd dp path
inkcherry Mar 22, 2026
fe420e8
pd+dp gsm8k pass
inkcherry Mar 22, 2026
2f8536c
update readme
inkcherry Mar 22, 2026
fd4ec66
move the mesh floder
inkcherry Mar 22, 2026
185be91
refactor
inkcherry Mar 22, 2026
7e28839
fix refactor
inkcherry Mar 22, 2026
96a8443
update ut
inkcherry Mar 22, 2026
c8efdee
update ut
inkcherry Mar 22, 2026
450be20
fix ci
inkcherry Mar 22, 2026
8b1dc6c
update
inkcherry Mar 22, 2026
30efcad
format
inkcherry Mar 23, 2026
e035f54
format
inkcherry Mar 23, 2026
f46ea41
style: apply black formatting to pass CI checks
inkcherry Mar 23, 2026
3d46cb5
fix dsv3 mtp&qwen thinking ci
inkcherry Mar 23, 2026
43124b3
Merge remote-tracking branch 'origin/main' into pr253
inkcherry Mar 23, 2026
0a07b36
fix mtp
inkcherry Mar 24, 2026
19f1125
Merge main into atom_pd + rename mesh -> kv_transfer
inkcherry Apr 17, 2026
45f4761
Fix test_scheduler 3-tuple and revert kv_transfer_params on chat path
inkcherry Apr 17, 2026
88507de
style: black format kv_transfer imports after rename
inkcherry Apr 17, 2026
b79f56b
support AINIC in mori pd
ZhangLirong-amd Apr 21, 2026
a6a0af9
Merge branch 'main' into atom_pd
ZhangLirong-amd Apr 21, 2026
e2f6a80
Apply suggestion from @Copilot
ZhangLirong-amd Apr 21, 2026
dc42fdd
Update model_runner.py
ZhangLirong-amd Apr 21, 2026
0410db3
reformat
ZhangLirong-amd Apr 21, 2026
58dfebb
ruff check
ZhangLirong-amd Apr 21, 2026
c4b7634
fix some scheduler
ZhangLirong-amd Apr 21, 2026
1ef6efb
revsert port change
ZhangLirong-amd Apr 21, 2026
cccce75
Merge branch 'main' into atom_pd
ZhangLirong-amd Apr 21, 2026
0abb64e
revert mla without pd
ZhangLirong-amd Apr 22, 2026
97a6fa8
revert mla
ZhangLirong-amd Apr 22, 2026
0f9f0b6
fix glm5 core dump
ZhangLirong-amd Apr 22, 2026
8e87561
Merge branch 'main' into atom_pd
ZhangLirong-amd Apr 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

**ATOM** (AiTer Optimized Model) is a lightweight vLLM-like implementation, focusing on integration and optimization based on [AITER](https://github.com/ROCm/aiter).

## 📢 News

- **[2026/03]** ATOM now supports **Prefill/Decode (P/D) disaggregation** — run prefill and decode on separate GPU nodes with RDMA-based KV cache transfer via [MORI-IO](https://github.com/ROCm/mori). See [disaggregation docs](atom/kv_transfer/disaggregation/README.md).

## 🚀 Features

- **ROCm Optimized**: Built on AMD's ROCm platform with [AITER](https://github.com/ROCm/aiter) kernels (ASM, CK, Triton)
Expand Down
13 changes: 13 additions & 0 deletions atom/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,7 @@ class Config:
enable_dp_attention: bool = False
torch_dtype: torch.dtype = field(init=False)
speculative_config: Optional[SpeculativeConfig] = None
kv_transfer_config: dict = field(default_factory=dict)

enable_tbo: bool = False
enable_tbo_decode: bool = False
Expand Down Expand Up @@ -894,6 +895,18 @@ def __post_init__(self):
else torch.bfloat16
)

if hasattr(self, "kv_transfer_config") and isinstance(
self.kv_transfer_config, str
):
import json

try:
self.kv_transfer_config = json.loads(self.kv_transfer_config)
except json.JSONDecodeError:
import ast

self.kv_transfer_config = ast.literal_eval(self.kv_transfer_config)

if self.speculative_config is not None:
num_spec = self.speculative_config.num_speculative_tokens
if num_spec is None or num_spec < 1 or num_spec > 4:
Expand Down
44 changes: 37 additions & 7 deletions atom/entrypoints/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,16 @@ def _send_stream_chunk_direct(
"finished_at": time.time(),
"started_at": started_at,
}
if getattr(request_output, "kv_transfer_params_output", None):
chunk_data["kv_transfer_params"] = request_output.kv_transfer_params_output
loop.call_soon_threadsafe(stream_queue.put_nowait, chunk_data)


async def generate_async(
prompt: str, sampling_params: SamplingParams, request_id: str
prompt: str,
sampling_params: SamplingParams,
request_id: str,
kv_transfer_params: Optional[Dict[str, Any]] = None,
) -> AsyncGenerator[Dict[str, Any], None]:
"""Generate text asynchronously for non-streaming requests."""
global engine, tokenizer
Expand All @@ -155,8 +160,13 @@ async def generate_async(
all_token_ids: List[int] = []
finish_reason: Optional[str] = None
seq = None
kv_transfer_output_meta_info = None

def completion_callback(request_output: RequestOutput):
nonlocal kv_transfer_output_meta_info
kv_transfer_output_meta_info = getattr(
request_output, "kv_transfer_params_output", None
)
now = time.time()
loop.call_soon_threadsafe(
token_queue.put_nowait,
Expand All @@ -170,7 +180,10 @@ def completion_callback(request_output: RequestOutput):

def do_preprocess():
return engine.io_processor.preprocess(
prompt, sampling_params, stream_callback=completion_callback
prompt,
sampling_params,
stream_callback=completion_callback,
kv_transfer_params=kv_transfer_params,
)

seq = await loop.run_in_executor(None, do_preprocess)
Expand Down Expand Up @@ -204,7 +217,7 @@ def do_preprocess():
else 0.0
)

yield {
response = {
"text": text,
"token_ids": all_token_ids,
"finish_reason": finish_reason,
Expand All @@ -214,6 +227,9 @@ def do_preprocess():
"tpot": tpot,
"latency": latency,
}
if kv_transfer_output_meta_info is not None:
response["kv_transfer_output_meta_info"] = kv_transfer_output_meta_info
yield response


def validate_model(requested_model: Optional[str]) -> None:
Expand All @@ -227,7 +243,10 @@ def validate_model(requested_model: Optional[str]) -> None:


async def setup_streaming_request(
prompt: str, sampling_params: SamplingParams, request_id: str
prompt: str,
sampling_params: SamplingParams,
request_id: str,
kv_transfer_params: Optional[Dict[str, Any]] = None,
) -> Tuple[int, asyncio.Queue]:
"""Set up a streaming request with the engine."""
global engine, _stream_queues, _seq_id_to_request_id
Expand All @@ -246,7 +265,10 @@ def stream_callback(request_output: RequestOutput) -> None:

def do_preprocess():
seq = engine.io_processor.preprocess(
prompt, sampling_params, stream_callback=stream_callback
prompt,
sampling_params,
stream_callback=stream_callback,
kv_transfer_params=kv_transfer_params,
)
_seq_id_to_request_id[seq.id] = request_id
return seq
Expand Down Expand Up @@ -427,7 +449,10 @@ async def completions(request: CompletionRequest):
# Streaming
if request.stream:
seq_id, stream_queue = await setup_streaming_request(
request.prompt, sampling_params, request_id
request.prompt,
sampling_params,
request_id,
kv_transfer_params=request.kv_transfer_params,
)
gen = stream_completion_response(
request_id,
Expand All @@ -445,7 +470,12 @@ async def completions(request: CompletionRequest):

# Non-streaming
final_output = None
async for output in generate_async(request.prompt, sampling_params, request_id):
async for output in generate_async(
request.prompt,
sampling_params,
request_id,
kv_transfer_params=request.kv_transfer_params,
):
final_output = output

if final_output is None:
Expand Down
4 changes: 4 additions & 0 deletions atom/entrypoints/openai/protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ class CompletionRequest(BaseModel):
stop: Optional[List[str]] = None
ignore_eos: Optional[bool] = False
stream: Optional[bool] = False
# Optional KV-transfer metadata for P/D disaggregation.
kv_transfer_params: Optional[Dict[str, Any]] = None


# ============================================================================
Expand Down Expand Up @@ -143,6 +145,8 @@ class CompletionResponse(BaseModel):
model: str
choices: List[Dict[str, Any]]
usage: Dict[str, Any]
# Optional KV-transfer metadata returned for P/D disaggregation.
kv_transfer_params: Optional[Dict[str, Any]] = None


class ModelCard(BaseModel):
Expand Down
16 changes: 15 additions & 1 deletion atom/entrypoints/openai/serving_completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def create_completion_chunk(
text: str,
finish_reason: Optional[str] = None,
usage: Optional[Dict] = None,
**extra_fields: Any,
) -> str:
"""Create a text completion chunk in SSE format."""
chunk = {
Expand All @@ -40,6 +41,7 @@ def create_completion_chunk(
}
],
}
chunk.update(extra_fields)
if usage is not None:
chunk["usage"] = usage
return f"data: {json.dumps(chunk)}\n\n"
Expand All @@ -63,11 +65,16 @@ async def stream_completion_response(
new_text = chunk_data["text"]
num_tokens_output += len(chunk_data.get("token_ids", []))

extra_fields: Dict[str, Any] = {}
if "kv_transfer_params" in chunk_data:
extra_fields["kv_transfer_params"] = chunk_data["kv_transfer_params"]

yield create_completion_chunk(
request_id,
model,
new_text,
finish_reason=chunk_data.get("finish_reason"),
**extra_fields,
)

if chunk_data.get("finished", False):
Expand Down Expand Up @@ -99,7 +106,7 @@ def build_completion_response(
final_output: Dict[str, Any],
) -> CompletionResponse:
"""Build a non-streaming text completion response."""
return CompletionResponse(
response = CompletionResponse(
id=request_id,
created=int(time.time()),
model=model,
Expand All @@ -120,3 +127,10 @@ def build_completion_response(
"latency_s": round(final_output.get("latency", 0.0), 4),
},
)
if "kv_transfer_output_meta_info" in final_output:
response = response.model_copy(
update={
"kv_transfer_params": final_output["kv_transfer_output_meta_info"],
}
)
return response
Empty file added atom/kv_transfer/__init__.py
Empty file.
Loading
Loading