Repository navigation
Give every launch script the same three subcommands #2431
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,7 +4,6 @@ | |
|
|
||
| import re | ||
| from dataclasses import dataclass | ||
| from functools import partial | ||
| from typing import Literal | ||
|
|
||
| import typer | ||
|
|
@@ -93,15 +92,15 @@ def _prepare_megatron_ckpt(args: ScriptArgs): | |
| ) | ||
|
|
||
|
|
||
| def _prepare_cp(args: ScriptArgs): | ||
| U.rsync_simple( | ||
| path_src=f"{args.model_dir}/{args.model_name}_torch_dist", | ||
| path_dst=f"{args.model_local_dir}/{args.model_name}_torch_dist", | ||
| ) | ||
| U.rsync_simple( | ||
| path_src=f"{args.model_dir}/{args.model_name}", | ||
| path_dst=f"{args.model_local_dir}/{args.model_name}", | ||
| ) | ||
| def _prepare_cmd(args: ScriptArgs) -> dict[str, str]: | ||
| copies = [ | ||
| U.rsync_cmd( | ||
| f"{args.model_dir}/{args.model_name}_torch_dist", | ||
| f"{args.model_local_dir}/{args.model_name}_torch_dist", | ||
| ), | ||
| U.rsync_cmd(f"{args.model_dir}/{args.model_name}", f"{args.model_local_dir}/{args.model_name}"), | ||
| ] | ||
| return {"trainer": " && ".join(copies)} | ||
|
|
||
|
|
||
| def _execute_train(args: ScriptArgs, before_ray_job_submit=None): | ||
|
|
@@ -299,27 +298,37 @@ def _execute_train(args: ScriptArgs, before_ray_job_submit=None): | |
|
|
||
| U.execute_train( | ||
| train_args=train_args, | ||
| config=args, | ||
| # TODO may get it from `config` | ||
| num_gpus_per_node=args.num_gpus_per_node, | ||
| megatron_model_type=args.megatron_model_type, | ||
| extra_env_vars={**sglang_extra_env_vars}, | ||
| megatron_path=args.megatron_path, | ||
| before_ray_job_submit=before_ray_job_submit, | ||
| prepare_cmd=_prepare_cmd(args), | ||
| ) | ||
|
|
||
|
|
||
| @app.command() | ||
| @U.dataclass_cli | ||
| def train(args: ScriptArgs): | ||
| def full_train(args: ScriptArgs) -> None: | ||
| _prepare_download(args) | ||
| _prepare_bf16_ckpt(args) | ||
| _execute_train(args, before_ray_job_submit=partial(_prepare_ray_dependent, args)) | ||
| _prepare_megatron_ckpt(args) | ||
| _execute_train(args) | ||
|
|
||
|
|
||
| def _prepare_ray_dependent(args: ScriptArgs): | ||
| @app.command() | ||
| @U.dataclass_cli | ||
| def prepare(args: ScriptArgs) -> None: | ||
| _prepare_download(args) | ||
| _prepare_bf16_ckpt(args) | ||
| _prepare_megatron_ckpt(args) | ||
| _prepare_cp(args) | ||
|
|
||
|
|
||
| @app.command() | ||
| @U.dataclass_cli | ||
| def train(args: ScriptArgs) -> None: | ||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Update every caller of the renamed launcher roles This split makes |
||
| _execute_train(args) | ||
|
|
||
|
|
||
| @app.callback() | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P1] Stage checkpoints on inference workers too
When
--cluster-backend kubernetesis used withmodel_local_dir != model_dir(the default here), this map attaches both rsyncs only to the trainer role. In deliver-2,helm_backend/launcher/values/pool_entry.py::_with_prepare_cmdskips inference-engine pods, whilecompute_engine_launch_cmdpasses the local HF path to SGLang asmodel_path; disaggregated rollout nodes therefore never receive the model, and colocated inference pods can start before the trainer's rsync completes. Add inference-side staging; the same pattern affects the V4/GLM launchers.