Repository navigation
Replace unittest skipTest from transformers with pytest.skip #4297
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 10 commits
743934d
63e46fa
8e5beee
dd1c100
c0e0ed4
c624342
8353d2b
cd8f566
4beb59f
d47cf9f
d2278c1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,12 +22,20 @@ | |
| import pytest | ||
| import torch | ||
| from transformers import is_bitsandbytes_available, is_comet_available, is_sklearn_available, is_wandb_available | ||
| from transformers.testing_utils import torch_device | ||
| from transformers.utils import is_peft_available, is_rich_available, is_vision_available | ||
| from transformers.testing_utils import backend_device_count, torch_device | ||
| from transformers.utils import ( | ||
| is_flash_attn_2_available, | ||
| is_kernels_available, | ||
| is_peft_available, | ||
| is_rich_available, | ||
| is_torch_available, | ||
| is_vision_available, | ||
| ) | ||
|
|
||
| from trl import BaseBinaryJudge, BasePairwiseJudge | ||
| from trl.import_utils import ( | ||
| is_joblib_available, | ||
| is_liger_kernel_available, | ||
| is_llm_blender_available, | ||
| is_math_verify_available, | ||
| is_mergekit_available, | ||
|
|
@@ -37,6 +45,7 @@ | |
|
|
||
| require_bitsandbytes = pytest.mark.skipif(not is_bitsandbytes_available(), reason="test requires bitsandbytes") | ||
| require_comet = pytest.mark.skipif(not is_comet_available(), reason="test requires comet_ml") | ||
| require_liger_kernel = pytest.mark.skipif(not is_liger_kernel_available(), reason="test requires liger-kernel") | ||
| require_llm_blender = pytest.mark.skipif(not is_llm_blender_available(), reason="test requires llm-blender") | ||
| require_math_latex = pytest.mark.skipif(not is_math_verify_available(), reason="test requires math_verify") | ||
| require_mergekit = pytest.mark.skipif(not is_mergekit_available(), reason="test requires mergekit") | ||
|
|
@@ -45,8 +54,15 @@ | |
| require_sklearn = pytest.mark.skipif( | ||
| not (is_sklearn_available() and is_joblib_available()), reason="test requires sklearn" | ||
| ) | ||
| require_torch_accelerator = pytest.mark.skipif( | ||
| torch_device is None or torch_device == "cpu", reason="test requires accelerator" | ||
| ) | ||
| require_torch_multi_accelerator = pytest.mark.skipif( | ||
| not is_torch_available() or backend_device_count(torch_device) <= 1, reason="test requires multiple accelerators" | ||
| ) | ||
| require_vision = pytest.mark.skipif(not is_vision_available(), reason="test requires vision") | ||
| require_vllm = pytest.mark.skipif(not is_vllm_available(), reason="test requires vllm") | ||
| require_wandb = pytest.mark.skipif(not is_wandb_available(), reason="test requires wandb") | ||
| require_no_wandb = pytest.mark.skipif(is_wandb_available(), reason="test requires no wandb") | ||
| require_3_accelerators = pytest.mark.skipif( | ||
| not (getattr(torch, torch_device, torch.cuda).device_count() >= 3), | ||
|
|
@@ -69,6 +85,20 @@ def is_bitsandbytes_multi_backend_available() -> bool: | |
| ) | ||
|
|
||
|
|
||
| # Function ported from transformers.testing_utils | ||
| def require_flash_attn(): | ||
| flash_attn_available = is_flash_attn_2_available() | ||
| kernels_available = is_kernels_available() | ||
| try: | ||
| from kernels import get_kernel | ||
|
|
||
| get_kernel("kernels-community/flash-attn") | ||
| except Exception: | ||
| kernels_available = False | ||
|
|
||
| return pytest.mark.skipif(not (kernels_available or flash_attn_available), reason="test requires Flash Attention") | ||
|
|
||
|
Comment on lines
+88
to
+103
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think this one it 100% correct, because if kernels is available, but not flash-attn, and you try to use
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I just ported it from transformers. 😅 def require_flash_attn(test_case):
flash_attn_available = is_flash_attn_2_available()
kernels_available = is_kernels_available()
try:
from kernels import get_kernel
get_kernel("kernels-community/flash-attn")
except Exception as _:
kernels_available = False
return unittest.skipUnless(kernels_available | flash_attn_available, "test requires Flash Attention")(test_case)Do you think I made an error in porting it? @qgallouedec
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The logic on
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK I see.
@albertvillanova we can do this in a future PR
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree! I created an issue for that: |
||
|
|
||
| class RandomBinaryJudge(BaseBinaryJudge): | ||
| """ | ||
| Random binary judge, for testing purposes. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
other wise we have to change our decorators to
@require_flash_attn()There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks a lot for your review, @kashif! 🤗
Finally, I aligned the implementation with the rest of the "require" pytest-markers, by calling an auxiliary function
is_x_available.