diff --git a/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_256k.py b/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_256k.py index f5513275..576da6ca 100644 --- a/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_256k.py +++ b/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_256k.py @@ -11,6 +11,7 @@ class Config(Parent): "miles_cfg.expert_model_parallel_size": 8, "miles_cfg.context_parallel_size": 4, "miles_cfg.align_sequences_to_parallel_layout": True, + "miles_cfg.max_tokens_per_gpu": 65536, "miles_cfg.cli_options.log_probs_chunk_size": -1, } diff --git a/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_64k.py b/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_64k.py index f7b77e6a..ffc7c2a0 100644 --- a/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_64k.py +++ b/src/spindle/configs/nemotron35_lightning_30b_a3b_lora_64k.py @@ -16,6 +16,7 @@ class Config(Parent): overrides = { "miles_cfg.expert_model_parallel_size": 2, "miles_cfg.max_lora_slots": 4, + "miles_cfg.max_tokens_per_gpu": 49152, "miles_cfg.cli_options.log_probs_chunk_size": 4096, "sglang_cfg.max_loras_per_batch": 4, "sglang_cfg.max_loaded_loras": 32,