Skip to content

Fix: rename 'eval_strategy' to 'evaluation_strategy' in TrainingArgum… #39109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
trevorhughdavis wants to merge 1 commit into from
Closed

Fix: rename 'eval_strategy' to 'evaluation_strategy' in TrainingArgum… #39109

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 22 additions & 21 deletions src/transformers/training_args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ class TrainingArguments:
Whether to run predictions on the test set or not. This argument is not directly used by [`Trainer`], it's
intended to be used by your training/evaluation scripts instead. See the [example
scripts](https://github.com/huggingface/transformers/tree/main/examples) for more details.
eval_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`):
evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`):

The evaluation strategy to adopt during training. Possible values are:

- `"no"`: No evaluation is done during training.
Expand Down Expand Up @@ -424,7 +425,7 @@ class TrainingArguments:
Whether to drop the last incomplete batch (if the length of the dataset is not divisible by the batch size)
or not.
eval_steps (`int` or `float`, *optional*):
Number of update steps between two evaluations if `eval_strategy="steps"`. Will default to the same
Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the same
value as `logging_steps` if not set. Should be an integer or a float in range `[0,1)`. If smaller than 1,
will be interpreted as ratio of total training steps.
dataloader_num_workers (`int`, *optional*, defaults to 0):
Expand Down Expand Up @@ -459,7 +460,7 @@ class TrainingArguments:

<Tip>

When set to `True`, the parameters `save_strategy` needs to be the same as `eval_strategy`, and in
When set to `True`, the parameters `save_strategy` needs to be the same as `evaluation_strategy`and in
the case it is "steps", `save_steps` must be a round multiple of `eval_steps`.

</Tip>
Expand Down Expand Up @@ -801,7 +802,7 @@ class TrainingArguments:
average_tokens_across_devices (`bool`, *optional*, defaults to `False`):
Whether or not to average tokens across devices. If enabled, will use all_reduce to synchronize
num_tokens_in_batch for precise loss calculation. Reference:
https://github.com/huggingface/transformers/issues/34242
httcps://github.com/huggingface/transformers/issues/34242
"""

# Sometimes users will pass in a `str` repr of a dict in the CLI
Expand Down Expand Up @@ -836,7 +837,7 @@ class TrainingArguments:
do_train: bool = field(default=False, metadata={"help": "Whether to run training."})
do_eval: bool = field(default=False, metadata={"help": "Whether to run eval on the dev set."})
do_predict: bool = field(default=False, metadata={"help": "Whether to run predictions on the test set."})
eval_strategy: Union[IntervalStrategy, str] = field(
evaluation_strategy: Union[IntervalStrategy, str] = field(
default="no",
metadata={"help": "The evaluation strategy to use."},
)
Expand Down Expand Up @@ -1592,14 +1593,14 @@ def __post_init__(self):
if self.disable_tqdm is None:
self.disable_tqdm = logger.getEffectiveLevel() > logging.WARN

if isinstance(self.eval_strategy, EvaluationStrategy):
if isinstance(self.evaluation_strategy, EvaluationStrategy):
warnings.warn(
"using `EvaluationStrategy` for `eval_strategy` is deprecated and will be removed in version 5"
" of 🤗 Transformers. Use `IntervalStrategy` instead",
FutureWarning,
)
# Go back to the underlying string or we won't be able to instantiate `IntervalStrategy` on it.
self.eval_strategy = self.eval_strategy.value
self.evaluation_strategy = self.evaluation_strategy.value
if self.no_cuda:
warnings.warn(
"using `no_cuda` is deprecated and will be removed in version 5.0 of 🤗 Transformers. "
Expand All @@ -1614,13 +1615,13 @@ def __post_init__(self):
FutureWarning,
)

self.eval_strategy = IntervalStrategy(self.eval_strategy)
self.evaluation_strategy = IntervalStrategy(self.evaluation_strategy)
self.logging_strategy = IntervalStrategy(self.logging_strategy)
self.save_strategy = SaveStrategy(self.save_strategy)
self.hub_strategy = HubStrategy(self.hub_strategy)

self.lr_scheduler_type = SchedulerType(self.lr_scheduler_type)
if self.do_eval is False and self.eval_strategy != IntervalStrategy.NO:
if self.do_eval is False and self.evaluation_strategy != IntervalStrategy.NO:
self.do_eval = True

if self.torch_empty_cache_steps is not None:
Expand All @@ -1630,13 +1631,13 @@ def __post_init__(self):
)

# eval_steps has to be defined and non-zero, fallbacks to logging_steps if the latter is non-zero
if self.eval_strategy == IntervalStrategy.STEPS and (self.eval_steps is None or self.eval_steps == 0):
if self.evaluation_strategy == IntervalStrategy.STEPS and (self.eval_steps is None or self.eval_steps == 0):
if self.logging_steps > 0:
logger.info(f"using `logging_steps` to initialize `eval_steps` to {self.logging_steps}")
self.eval_steps = self.logging_steps
else:
raise ValueError(
f"evaluation strategy {self.eval_strategy} requires either non-zero --eval_steps or"
f"evaluation strategy {self.evaluation_strategy} requires either non-zero --eval_steps or"
" --logging_steps"
)

Expand All @@ -1648,7 +1649,7 @@ def __post_init__(self):
if self.logging_steps != int(self.logging_steps):
raise ValueError(f"--logging_steps must be an integer if bigger than 1: {self.logging_steps}")
self.logging_steps = int(self.logging_steps)
if self.eval_strategy == IntervalStrategy.STEPS and self.eval_steps > 1:
if self.evaluation_strategy == IntervalStrategy.STEPS and self.eval_steps > 1:
if self.eval_steps != int(self.eval_steps):
raise ValueError(f"--eval_steps must be an integer if bigger than 1: {self.eval_steps}")
self.eval_steps = int(self.eval_steps)
Expand All @@ -1659,12 +1660,12 @@ def __post_init__(self):

# Sanity checks for load_best_model_at_end: we require save and eval strategies to be compatible.
if self.load_best_model_at_end and self.save_strategy != SaveStrategy.BEST:
if self.eval_strategy != self.save_strategy:
if self.evaluation_strategy != self.save_strategy:
raise ValueError(
"--load_best_model_at_end requires the save and eval strategy to match, but found\n- Evaluation "
f"strategy: {self.eval_strategy}\n- Save strategy: {self.save_strategy}"
f"strategy: {self.evaluation_strategy}\n- Save strategy: {self.save_strategy}"
)
if self.eval_strategy == IntervalStrategy.STEPS and self.save_steps % self.eval_steps != 0:
if self.evaluation_strategy == IntervalStrategy.STEPS and self.save_steps % self.eval_steps != 0:
if self.eval_steps < 1 or self.save_steps < 1:
if not (self.eval_steps < 1 and self.save_steps < 1):
raise ValueError(
Expand Down Expand Up @@ -1749,7 +1750,7 @@ def __post_init__(self):
)

if self.lr_scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
if self.eval_strategy == IntervalStrategy.NO:
if self.evaluation_strategy == IntervalStrategy.NO:
raise ValueError("lr_scheduler_type reduce_lr_on_plateau requires an eval strategy")
if not is_torch_available():
raise ValueError("lr_scheduler_type reduce_lr_on_plateau requires torch>=0.2.0")
Expand Down Expand Up @@ -2687,7 +2688,7 @@ def set_evaluate(
but requires more memory).
delay (`float`, *optional*):
Number of epochs or steps to wait for before the first evaluation can be performed, depending on the
eval_strategy.
evaluation_strategy.
loss_only (`bool`, *optional*, defaults to `False`):
Ignores all outputs except the loss.
jit_mode (`bool`, *optional*):
Expand All @@ -2704,10 +2705,10 @@ def set_evaluate(
100
```
"""
self.eval_strategy = IntervalStrategy(strategy)
if self.eval_strategy == IntervalStrategy.STEPS and steps == 0:
raise ValueError("Setting `strategy` as 'steps' requires a positive value for `steps`.")
self.do_eval = self.eval_strategy != IntervalStrategy.NO
self.evaluation_strategy = IntervalStrategy(strategy)
if self.evaluation_strategy == IntervalStrategy.STEPS and steps == 0:
raise ValueError("Setting `evaluation_stategy` as 'steps' requires a positive value for `steps`.")
self.do_eval = self.evaluation_strategy != IntervalStrategy.NO
self.eval_steps = steps
self.per_device_eval_batch_size = batch_size
self.eval_accumulation_steps = accumulation_steps
Expand Down