We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 02776d2 commit 51083d1Copy full SHA for 51083d1
docs/source/en/deepspeed.md
@@ -1091,7 +1091,7 @@ ds_config = {
1091
"overlap_comm": True,
1092
"contiguous_gradients": True,
1093
"reduce_bucket_size": model_hidden_size * model_hidden_size,
1094
- "stage3_prefetch_bucket_size": 0.9 * model_hidden_size * model_hidden_size,
+ "stage3_prefetch_bucket_size": 3774873,
1095
"stage3_param_persistence_threshold": 10 * model_hidden_size
1096
},
1097
"steps_per_print": 2000,
0 commit comments