Skip to content
Merged
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions src/transformers/training_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,19 @@ class OptimizerNames(ExplicitEnum):
GALORE_ADAFACTOR_LAYERWISE = "galore_adafactor_layerwise"


# Sometimes users will pass in a `str` repr of a dict in the CLI
# We need to track what fields those can be. Each time a new arg
# has a dict type, it must be added to this list.
# Important: These should be typed with Optional[Union[dict,str,...]]
Comment thread
muellerzr marked this conversation as resolved.
Resolved
VALID_DICT_FIELDS = [
Comment thread
muellerzr marked this conversation as resolved.
Resolved
"accelerator_config",
"fsdp_config",
"deepspeed",
"gradient_checkpointing_kwargs",
"lr_scheduler_kwargs",
]


# TODO: `TrainingArguments` users rely on it being fully mutable. In the future see if we can narrow this to a few keys: https://github.com/huggingface/transformers/pull/25903
@dataclass
class TrainingArguments:
Expand Down Expand Up @@ -803,11 +816,11 @@ class TrainingArguments:
default="linear",
metadata={"help": "The scheduler type to use."},
)
lr_scheduler_kwargs: Optional[Dict] = field(
lr_scheduler_kwargs: Optional[Union[dict, str]] = field(
default_factory=dict,
metadata={
"help": (
"Extra parameters for the lr_scheduler such as {'num_cycles': 1} for the cosine with hard restarts"
"Extra parameters for the lr_scheduler such as {'num_cycles': 1} for the cosine with hard restarts."
)
},
)
Expand Down Expand Up @@ -1118,7 +1131,6 @@ class TrainingArguments:
)
},
)
# Do not touch this type annotation or it will stop working in CLI
fsdp_config: Optional[Union[dict, str]] = field(
default=None,
metadata={
Expand All @@ -1137,8 +1149,7 @@ class TrainingArguments:
)
},
)
# Do not touch this type annotation or it will stop working in CLI
accelerator_config: Optional[str] = field(
accelerator_config: Optional[Union[dict, str]] = field(
default=None,
metadata={
"help": (
Expand All @@ -1147,8 +1158,7 @@ class TrainingArguments:
)
},
)
# Do not touch this type annotation or it will stop working in CLI
deepspeed: Optional[str] = field(
deepspeed: Optional[Union[dict, str]] = field(
default=None,
metadata={
"help": (
Expand Down Expand Up @@ -1252,7 +1262,7 @@ class TrainingArguments:
"help": "If True, use gradient checkpointing to save memory at the expense of slower backward pass."
},
)
gradient_checkpointing_kwargs: Optional[dict] = field(
gradient_checkpointing_kwargs: Optional[Union[dict, str]] = field(
default=None,
metadata={
"help": "Gradient checkpointing key word arguments such as `use_reentrant`. Will be passed to `torch.utils.checkpoint.checkpoint` through `model.gradient_checkpointing_enable`."
Expand Down Expand Up @@ -1380,6 +1390,14 @@ class TrainingArguments:
)

def __post_init__(self):
# Parse in args that could be `dict` sent in from the CLI as a string
for field in VALID_DICT_FIELDS:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like just checking for a few possible args!

passed_value = getattr(self, field)
# We only want to do this if the str starts with a bracket to indiciate a `dict`
# else its likely a filename if supported
if isinstance(passed_value, str) and passed_value.startswith("{"):
setattr(self, field, json.loads(passed_value))

# expand paths, if not os.makedirs("~/bar") will make directory
# in the current directory instead of the actual home
# see https://github.com/huggingface/transformers/issues/10628
Expand Down