Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/transformers/cli/add_new_model_like.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from collections.abc import Callable
from datetime import date
from pathlib import Path
from typing import Annotated, Any
from typing import Annotated, Any, cast

import typer

Expand Down Expand Up @@ -57,10 +57,11 @@ def visit_SimpleStatementLine(self, node: cst.SimpleStatementLine):
body=[m.Assign(targets=[m.AssignTarget(target=m.Name())])]
)
if not self.is_in_class and m.matches(node, simple_top_level_assign_structure):
assigned_variable = node.body[0].targets[0].target.value
stmt = cast(cst.Assign, node.body[0])
assigned_variable = cast(cst.Name, stmt.targets[0].target).value
if assigned_variable == "__all__":
elements = node.body[0].value.elements
self.public_classes = [element.value.value for element in elements]
elements = cast(cst.Tuple, stmt.value).elements
self.public_classes = [cast(cst.SimpleString, element.value).value for element in elements]


CURRENT_YEAR = date.today().year
Expand Down Expand Up @@ -316,7 +317,10 @@ def insert_model_in_doc_toc(
with open(toc_file, "r") as f:
content = f.read()

old_model_toc = re.search(rf"- local: model_doc/{old_lowercase_name}\n {{8}}title: .*?\n", content).group(0)
toc_match = re.search(rf"- local: model_doc/{old_lowercase_name}\n {{8}}title: .*?\n", content)
if toc_match is None:
raise ValueError(f"Could not find TOC entry for {old_lowercase_name}")
old_model_toc = toc_match.group(0)
new_toc = f" - local: model_doc/{new_lowercase_name}\n title: {new_model_paper_name}\n"
add_content_to_file(
repo_path / "docs" / "source" / "en" / "_toctree.yml", new_content=new_toc, add_after=old_model_toc
Expand Down Expand Up @@ -378,7 +382,7 @@ def find_all_classes_from_file(module_name: str) -> set:


def find_modular_structure(
module_name: str, old_model_infos: ModelInfos, new_cased_name: str
module_name: Path, old_model_infos: ModelInfos, new_cased_name: str
) -> tuple[str, str, list]:
"""
Extract the modular structure that will be needed to copy a file `module_name` using modular.
Expand Down
6 changes: 4 additions & 2 deletions src/transformers/cli/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import re
import string
import time
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Awaitable
from typing import Annotated, Any
from urllib.parse import urljoin, urlparse

Expand Down Expand Up @@ -110,7 +110,9 @@ def __init__(self, model_id: str, user_id: str, base_url: str):
self.user_id = user_id
self.base_url = base_url

async def stream_output(self, stream: AsyncIterator[ChatCompletionStreamOutput]) -> tuple[str, str | Any | None]:
async def stream_output(
self, stream: Awaitable[AsyncIterator[ChatCompletionStreamOutput]]
) -> tuple[str, str | Any | None]:
self._console.print(f"[bold blue]<{self.model_id}>:")
with Live(console=self._console, refresh_per_second=4) as live:
text = ""
Expand Down
7 changes: 6 additions & 1 deletion src/transformers/cli/serving/chat_completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from .utils import (
BaseGenerateManager,
BaseHandler,
CBGenerateManager,
ToolCallParser,
_StreamError,
detect_tool_format,
Expand Down Expand Up @@ -120,11 +121,15 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse
tokenize=True,
)
if not use_cb:
from transformers import BatchEncoding
Comment thread
SunMarc marked this conversation as resolved.
Resolved

if not isinstance(inputs, BatchEncoding):
raise TypeError("Expected BatchEncoding from apply_chat_template with return_tensors='pt'")
inputs = inputs.to(model.device)

gen_config = self._build_generation_config(body, model.generation_config, use_cb=use_cb)
# TODO: remove when CB supports per-request generation config
if use_cb:
if use_cb and isinstance(gen_manager, CBGenerateManager):
gen_manager.init_cb(model, gen_config)

# Detect tool support for the loaded model
Expand Down
11 changes: 8 additions & 3 deletions src/transformers/cli/serving/response.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
from .utils import (
BaseGenerateManager,
BaseHandler,
CBGenerateManager,
ToolCallParser,
_StreamError,
detect_tool_format,
Expand Down Expand Up @@ -128,11 +129,15 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse
tokenize=True,
)
if not use_cb:
from transformers import BatchEncoding
Comment thread
SunMarc marked this conversation as resolved.
Resolved

if not isinstance(inputs, BatchEncoding):
raise TypeError("Expected BatchEncoding from apply_chat_template with return_tensors='pt'")
inputs = inputs.to(model.device)

gen_config = self._build_generation_config(body, model.generation_config, use_cb=use_cb)
# TODO: remove when CB supports per-request generation config
if use_cb:
if use_cb and isinstance(gen_manager, CBGenerateManager):
Comment thread
vasqu marked this conversation as resolved.
Resolved
gen_manager.init_cb(model, gen_config)
tool_format = detect_tool_format(model) if body.get("tools") else None

Expand Down Expand Up @@ -439,7 +444,7 @@ async def event_stream() -> AsyncGenerator[str, None]:
status="completed",
role="assistant",
content=[output_text_part],
annotations=[],
annotations=[], # type: ignore[call-arg]
)
yield self.chunk_to_sse(
ResponseOutputItemDoneEvent(
Expand Down Expand Up @@ -496,7 +501,7 @@ async def _non_streaming(
status="completed",
role="assistant",
content=[ResponseOutputText(type="output_text", text=full_text, annotations=[])],
annotations=[],
annotations=[], # type: ignore[call-arg]
)
]

Expand Down
14 changes: 11 additions & 3 deletions src/transformers/cli/serving/transcription.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def __init__(self, model_manager: ModelManager, generation_state: GenerationStat

def _validate_request(self, form_keys: set[str]) -> None:
"""Validate transcription request fields."""
unexpected = form_keys - TransformersTranscriptionCreateParams.__mutable_keys__
unexpected = form_keys - getattr(TransformersTranscriptionCreateParams, "__mutable_keys__", set())
if unexpected:
raise HTTPException(status_code=422, detail=f"Unexpected fields in the request: {unexpected}")
unused = form_keys & UNUSED_TRANSCRIPTION_FIELDS
Expand Down Expand Up @@ -103,13 +103,21 @@ async def handle_request(self, request: Request) -> JSONResponse | StreamingResp

async with request.form() as form:
self._validate_request(set(form.keys()))
file_bytes = await form["file"].read()
file_field = form["file"]
if isinstance(file_field, str):
raise HTTPException(status_code=422, detail="Expected file upload, got string")
file_bytes = await file_field.read()
model = form["model"]
if not isinstance(model, str):
raise HTTPException(status_code=422, detail="Expected model name as string")
stream = str(form.get("stream", "false")).lower() == "true"

model_id_and_revision = self.model_manager.process_model_name(model)
audio_model, audio_processor = self.model_manager.load_model_and_processor(model_id_and_revision)
gen_manager = self.generation_state.get_manager(model_id_and_revision)
base_manager = self.generation_state.get_manager(model_id_and_revision)
if not isinstance(base_manager, GenerateManager):
raise HTTPException(status_code=400, detail="Audio transcription requires sequential generation (not CB)")
gen_manager = base_manager
audio_inputs = self._prepare_audio_inputs(file_bytes, audio_processor, audio_model)

if stream:
Expand Down
28 changes: 19 additions & 9 deletions src/transformers/cli/serving/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ def make_progress_tqdm_class(callback: Callable, model_id: str) -> type:

download_aggregator = DownloadAggregator(callback, model_id)

class ProgressTqdm(base_tqdm):
class ProgressTqdm(base_tqdm): # type: ignore[misc]
def __init__(self, *args, **kwargs):
self.sse_unit = kwargs.get("unit") or "it"
kwargs["disable"] = True
Expand Down Expand Up @@ -519,7 +519,7 @@ def generate_streaming(
"""

@abstractmethod
def generate_non_streaming(
async def generate_non_streaming(
self,
model: "PreTrainedModel",
processor: "ProcessorMixin | PreTrainedTokenizerFast",
Expand Down Expand Up @@ -562,7 +562,7 @@ def generate_streaming(
"""Start streaming generation via ``model.generate()`` on the inference thread."""
loop = asyncio.get_running_loop()
queue: asyncio.Queue = asyncio.Queue()
streamer = DirectStreamer(processor._tokenizer, loop, queue, skip_special_tokens=True)
streamer = DirectStreamer(processor._tokenizer, loop, queue, skip_special_tokens=True) # type: ignore[union-attr]
gen_kwargs = {**inputs, "streamer": streamer, "generation_config": gen_config, "tokenizer": processor}

def _run() -> None:
Expand Down Expand Up @@ -650,18 +650,22 @@ def generate_streaming(
request_id: str,
) -> tuple[asyncio.Queue, CBStreamer]:
"""Start streaming CB generation. Registers a per-request output handler."""
cb = self._cb
if cb is None:
raise RuntimeError("CB manager not initialized. Call init_cb() first.")
Comment thread
SunMarc marked this conversation as resolved.
Resolved

loop = asyncio.get_running_loop()
text_queue: asyncio.Queue = asyncio.Queue()

input_ids = inputs["input_ids"]
request_id = self._cb.add_request(
request_id = cb.add_request(
input_ids,
request_id=request_id,
streaming=True,
max_new_tokens=gen_config.max_new_tokens,
eos_token_id=gen_config.eos_token_id,
)
streamer = CBStreamer(self._cb, request_id, processor._tokenizer, loop, text_queue)
streamer = CBStreamer(cb, request_id, processor._tokenizer, loop, text_queue) # type: ignore[union-attr]

# Register a direct callback: the dispatcher calls this on the event loop with each GenerationOutput.
# This decodes tokens and pushes text straight to the SSE text_queue
Expand All @@ -673,7 +677,7 @@ def _on_output(output):
except Exception as e:
text_queue.put_nowait(_StreamError(str(e)))

self._cb.register_result_handler(request_id, _on_output)
cb.register_result_handler(request_id, _on_output)
return text_queue, streamer

async def generate_non_streaming(
Expand All @@ -685,6 +689,10 @@ async def generate_non_streaming(
request_id: str,
) -> tuple[str, int, list[int]]:
"""Run non-streaming CB generation. Registers a handler that resolves an asyncio.Future on completion."""
cb = self._cb
if cb is None:
raise RuntimeError("CB manager not initialized. Call init_cb() first.")
Comment thread
SunMarc marked this conversation as resolved.
Resolved

input_ids = inputs["input_ids"]
input_len = len(input_ids)

Expand All @@ -696,9 +704,9 @@ def _on_result(result):
if not future.done():
future.set_result(result)

self._cb.register_result_handler(request_id, _on_result)
cb.register_result_handler(request_id, _on_result)

self._cb.add_request(
cb.add_request(
input_ids,
request_id=request_id,
max_new_tokens=gen_config.max_new_tokens,
Expand All @@ -715,6 +723,8 @@ def _on_result(result):
@property
def scheduler(self) -> "Scheduler":
"""The CB scheduler (for testing/monitoring)."""
if self._cb is None:
raise RuntimeError("CB manager not initialized.")
return self._cb.batch_processor.scheduler

def stop(self) -> None:
Expand Down Expand Up @@ -829,7 +839,7 @@ def _validate_request(self, body: dict) -> None:

input_keys = set(body.keys())
if self._valid_params_class is not None:
unexpected = input_keys - self._valid_params_class.__mutable_keys__
unexpected = input_keys - getattr(self._valid_params_class, "__mutable_keys__", set())
if unexpected:
raise HTTPException(status_code=422, detail=f"Unexpected fields in the request: {unexpected}")
unused = input_keys & self._unused_fields
Expand Down
9 changes: 5 additions & 4 deletions src/transformers/cli/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def env(
"""Print information about the environment."""
import safetensors

safetensors_version = safetensors.__version__
safetensors_version = safetensors.__version__ if hasattr(safetensors, "__version__") else "unknown"
Comment thread
SunMarc marked this conversation as resolved.
Resolved

accelerate_version = "not installed"
accelerate_config = accelerate_config_str = "not found"
Expand Down Expand Up @@ -114,13 +114,14 @@ def env(
elif pt_xpu_available:
info["Using XPU in script?"] = "<fill in>"
info["XPU type"] = torch.xpu.get_device_name()
elif pt_hpu_available:
elif pt_hpu_available and hasattr(torch, "hpu"):
info["Using HPU in script?"] = "<fill in>"
info["HPU type"] = torch.hpu.get_device_name()
elif pt_npu_available:
elif pt_npu_available and hasattr(torch, "npu"):
info["Using NPU in script?"] = "<fill in>"
info["NPU type"] = torch.npu.get_device_name()
info["CANN version"] = torch.version.cann
if hasattr(torch.version, "cann"):
info["CANN version"] = torch.version.cann

print("\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n")
print(_format_dict(info))
Expand Down
8 changes: 8 additions & 0 deletions src/transformers/processing_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,14 @@ class ProcessorMixin(PushToHubMixin):
This is a mixin used to provide saving/loading functionality for all processor classes.
"""

# Dynamically set sub-processor attributes. Not every processor has all of these;
# they are populated via setattr in __init__ based on each subclass's `attributes`.
tokenizer: Any
feature_extractor: Any
image_processor: Any
video_processor: Any
chat_template: str | dict[str, str] | None

# Names need to be attr_class for attr in attributes
_auto_class = None
valid_processor_kwargs = ProcessingKwargs
Expand Down
25 changes: 19 additions & 6 deletions src/transformers/tokenization_utils_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@
from collections.abc import Callable, Collection, Mapping, Sequence, Sized
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, NamedTuple, Union
from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Union, overload

import numpy as np
from huggingface_hub import create_repo, is_offline_mode, list_repo_files
from packaging import version
from typing_extensions import TypeVar

from . import __version__
from .dynamic_module_utils import custom_object_save
Expand Down Expand Up @@ -188,7 +189,10 @@ class TokenSpan(NamedTuple):
end: int


class BatchEncoding(UserDict):
_V = TypeVar("_V", default=Any)


class BatchEncoding(UserDict, Generic[_V]):
"""
Holds the output of the [`~tokenization_utils_base.PreTrainedTokenizerBase.__call__`],
[`~tokenization_utils_base.PreTrainedTokenizerBase.encode_plus`] and
Expand Down Expand Up @@ -248,7 +252,16 @@ def n_sequences(self) -> int | None:
"""
return self._n_sequences

def __getitem__(self, item: int | str) -> Any | EncodingFast:
@overload
def __getitem__(self, item: str) -> _V: ...

@overload
def __getitem__(self, item: int) -> EncodingFast: ...

@overload
def __getitem__(self, item: slice) -> dict[str, _V]: ...

def __getitem__(self, item: int | str | slice) -> _V | EncodingFast | dict[str, _V]:
"""
If the key is a string, returns the value of the dict associated to `key` ('input_ids', 'attention_mask',
etc.).
Expand Down Expand Up @@ -753,7 +766,7 @@ def as_tensor(value, dtype=None):

return self

def to(self, device: str | torch.device, *, non_blocking: bool = False) -> BatchEncoding:
def to(self, device: str | torch.device, *, non_blocking: bool = False) -> BatchEncoding[torch.Tensor]:
"""
Send all values to device by calling `v.to(device, non_blocking=non_blocking)` (PyTorch only).

Expand Down Expand Up @@ -1291,7 +1304,7 @@ def __getattr__(self, key):

if key not in self.__dict__:
raise AttributeError(f"{self.__class__.__name__} has no attribute {key}")
return super().__getattr__(key)
return self.__dict__[key]
Comment thread
SunMarc marked this conversation as resolved.
Resolved

def get_special_tokens_mask(
self, token_ids_0: list[int], token_ids_1: list[int] | None = None, already_has_special_tokens: bool = False
Expand Down Expand Up @@ -2003,7 +2016,7 @@ def save_pretrained(

if push_to_hub:
commit_message = kwargs.pop("commit_message", None)
repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
repo_id = kwargs.pop("repo_id", str(save_directory).split(os.path.sep)[-1])
repo_id = create_repo(repo_id, exist_ok=True, **kwargs).repo_id
files_timestamps = self._get_files_timestamps(save_directory)

Expand Down
2 changes: 2 additions & 0 deletions utils/check_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
# the explicitly targeted paths but not transitive dependencies.
"file_globs": [
"src/transformers/_typing.py",
"src/transformers/cli/**/*.py",
"src/transformers/utils/**/*.py",
"src/transformers/generation/**/*.py",
"src/transformers/quantizers/**/*.py",
],
"check_args": [
"src/transformers/_typing.py",
"src/transformers/cli",
"src/transformers/utils",
"src/transformers/generation",
"src/transformers/quantizers",
Expand Down
Loading