|
| 1 | +# specify the default per-component configs |
| 2 | +defaults: |
| 3 | + # <folder_name>@<field_name>.<field_name>: <yaml_file_name> |
| 4 | + # actor_rollout_ref.actor: trainer/config/actor/veomni_actor.yaml |
| 5 | + - actor@actor_rollout_ref.actor: veomni_actor |
| 6 | + # data: trainer/config/data/legacy_data.yaml |
| 7 | + - data@data: legacy_data |
| 8 | + # (Rule-based) Reward manager config. |
| 9 | + - reward_manager@reward_manager |
| 10 | + # load the reference default config, then apply the fields in the current yaml |
| 11 | + # Reference model config. |
| 12 | + # Reference model will be enabled when actor.use_kl_loss or/and algorithm.use_kl_in_reward is/are True. |
| 13 | + - ref@actor_rollout_ref.ref: veomni_ref |
| 14 | + # Rollout model config. |
| 15 | + - rollout@actor_rollout_ref.rollout: rollout |
| 16 | + # Model config. |
| 17 | + - model@actor_rollout_ref.model: hf_model |
| 18 | + # Critic model config. |
| 19 | + - critic@critic: veomni_critic |
| 20 | + # Reward model config. |
| 21 | + - reward_model@reward_model: veomni_reward_loop |
| 22 | + # Rollout correction config. |
| 23 | + - algorithm@algorithm.rollout_correction: rollout_correction |
| 24 | + - _self_ |
| 25 | + |
| 26 | +actor_rollout_ref: |
| 27 | + hybrid_engine: True |
| 28 | + |
| 29 | + nccl_timeout: 600 # seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using veomni |
| 30 | + |
| 31 | + model: |
| 32 | + override_config: |
| 33 | + model_config: {} |
| 34 | + moe_config: |
| 35 | + freeze_moe_router: False |
| 36 | + |
| 37 | + use_fused_kernels: False # Whether to use custom fused kernels (PostProcessing, for memory efficiency) |
| 38 | + |
| 39 | + trust_remote_code: False |
| 40 | + |
| 41 | + # Whether to remove padding tokens in inputs during training |
| 42 | + use_remove_padding: false |
| 43 | + |
| 44 | + rollout: |
| 45 | + quantization: null |
| 46 | + |
| 47 | + layer_name_map: |
| 48 | + qkv_layer_name: qkv |
| 49 | + gate_proj_layer_name: gate_up |
| 50 | + |
| 51 | +custom_reward_function: |
| 52 | + path: null |
| 53 | + name: compute_score |
| 54 | + |
| 55 | +algorithm: |
| 56 | + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs |
| 57 | + _target_: verl.trainer.config.AlgoConfig |
| 58 | + gamma: 1.0 |
| 59 | + lam: 1.0 |
| 60 | + adv_estimator: gae |
| 61 | + norm_adv_by_std_in_grpo: True |
| 62 | + use_kl_in_reward: False |
| 63 | + kl_penalty: kl # how to estimate kl divergence |
| 64 | + kl_ctrl: |
| 65 | + # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs |
| 66 | + _target_: verl.trainer.config.KLControlConfig |
| 67 | + type: fixed |
| 68 | + kl_coef: 0.001 |
| 69 | + horizon: 10000 |
| 70 | + target_kl: 0.1 |
| 71 | + use_pf_ppo: False |
| 72 | + pf_ppo: |
| 73 | + reweight_method: pow # ["pow", "max_min", "max_random"] |
| 74 | + weight_pow: 2.0 |
| 75 | + |
| 76 | +trainer: |
| 77 | + balance_batch: True |
| 78 | + total_epochs: 30 |
| 79 | + total_training_steps: null |
| 80 | + project_name: verl_examples |
| 81 | + experiment_name: gsm8k |
| 82 | + logger: ["console", "wandb"] |
| 83 | + log_val_generations: 0 |
| 84 | + nnodes: 1 |
| 85 | + n_gpus_per_node: 8 |
| 86 | + save_freq: -1 |
| 87 | + esi_redundant_time: 0 |
| 88 | + |
| 89 | + # auto: find the last ckpt to resume. If can't find, start from scratch |
| 90 | + resume_mode: auto # or disable or resume_path if resume_from_path is set |
| 91 | + resume_from_path: null |
| 92 | + del_local_ckpt_after_load: False |
| 93 | + val_before_train: True |
| 94 | + test_freq: -1 |
| 95 | + critic_warmup: 0 |
| 96 | + default_hdfs_dir: null |
| 97 | + default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} |
| 98 | + max_actor_ckpt_to_keep: null |
| 99 | + max_critic_ckpt_to_keep: null |
| 100 | + # The timeout for ray worker group to wait for the register center to be ready |
| 101 | + ray_wait_register_center_timeout: 300 |
| 102 | + device: cuda |
| 103 | + # Directory for logging rollout data; no dump if null |
| 104 | + rollout_data_dir: null |
| 105 | + |
| 106 | + # whether to use legacy worker implementation |
| 107 | + # mode: "auto", "enable", or "disable" |
| 108 | + use_legacy_worker_impl: auto |
| 109 | + |
| 110 | +global_profiler: |
| 111 | + _target_: verl.utils.profiler.ProfilerConfig |
| 112 | + tool: null # choose between nsys, npu, torch, torch_memory |
| 113 | + steps: null # profile steps |
| 114 | + profile_continuous_steps: False |
| 115 | + save_path: "outputs/profile" # profiler saving path |
| 116 | + # Specific tool configs, can use +profiler.tool_config.[tool].xxx to config |
| 117 | + global_tool_config: |
| 118 | + # nsys config |
| 119 | + nsys: |
| 120 | + # True for each task has its own database, False for all tasks in one training step share one database. |
| 121 | + discrete: False |
| 122 | + |
| 123 | + # controller Nvidia Nsight Systems Options. Must set when profile_steps is not None. |
| 124 | + ## reference https://docs.nvidia.com/nsight-systems/UserGuide/index.html |
| 125 | + ## reference https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html |
| 126 | + controller_nsight_options: |
| 127 | + # Select the API(s) to be traced. |
| 128 | + trace: "cuda,nvtx,cublas,ucx" |
| 129 | + |
| 130 | + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". |
| 131 | + cuda-memory-usage: "true" |
| 132 | + |
| 133 | + # CUDA graphs will be traced as a whole |
| 134 | + cuda-graph-trace: "graph" |
| 135 | + |
| 136 | + # worker Nvidia Nsight Systems Options. Must set when profile_steps is not None. |
| 137 | + worker_nsight_options: |
| 138 | + # Select the API(s) to be traced. |
| 139 | + trace: "cuda,nvtx,cublas,ucx" |
| 140 | + |
| 141 | + # Track the GPU memory usage by CUDA kernels. Must be string type "true" or "false". |
| 142 | + cuda-memory-usage: "true" |
| 143 | + |
| 144 | + # CUDA graphs will be traced as a whole |
| 145 | + cuda-graph-trace: "graph" |
| 146 | + |
| 147 | + # Profiling only in a range of torch.cuda.profiler.start and stop. Do not change this config. |
| 148 | + capture-range: "cudaProfilerApi" |
| 149 | + |
| 150 | + # Specify the desired behavior when a capture range ends. |
| 151 | + # In verl we need the torch.cuda.profiler.start/stop pair to repeats n times. |
| 152 | + # valid values are "repeat-shutdown:n" or null. |
| 153 | + # For normal whole step profiling, n = len(profile_steps); |
| 154 | + # but for discrete profiling, n = len(profile_steps) * Number(subtasks). |
| 155 | + # Or you can just leave it null and the program will use n = len(profile_steps) * 6; |
| 156 | + capture-range-end: null |
| 157 | + |
| 158 | + # Send signal to the target application's process group. We let the program to exit by itself. |
| 159 | + kill: none |
| 160 | + |
| 161 | + # enable memory visualization for debugging memory usage |
| 162 | + torch_memory: |
| 163 | + # Maximum number of allocation entries to record |
| 164 | + trace_alloc_max_entries: 100_000 |
| 165 | + # The depth of the call stack to capture for each allocation |
| 166 | + stack_depth: 32 |
| 167 | + # 'alloc': records only allocation events || 'state': records memory state changes || 'all': records both. |
| 168 | + context: "all" |
| 169 | + # 'python': records Python stacks || 'cpp': records C++ stacks (available in some versions) || 'all': records both. |
| 170 | + stacks: "all" |
| 171 | + # devices, record_context etc. |
| 172 | + kw_args: {} |
| 173 | + |
| 174 | +# configs for TransferQueue |
| 175 | +transfer_queue: |
| 176 | + # Whether to enable transfer queue |
| 177 | + enable: False |
| 178 | + |
| 179 | +ray_kwargs: |
| 180 | + ray_init: |
| 181 | + num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then. |
| 182 | + timeline_json_file: null |
| 183 | + |
0 commit comments