From 01c8d12a78206778846de75fd8be8ce24ffc8f86 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 27 Jan 2025 17:14:21 -0700 Subject: [PATCH 01/48] added phi3_5 to transformers\models with .py files from original --- .../models/phi3_5/configuration_phi3_v.py | 218 ++ .../models/phi3_5/modeling_phi3_v.py | 1942 +++++++++++++++++ .../models/phi3_5/processing_phi3_v.py | 478 ++++ 3 files changed, 2638 insertions(+) create mode 100644 src/transformers/models/phi3_5/configuration_phi3_v.py create mode 100644 src/transformers/models/phi3_5/modeling_phi3_v.py create mode 100644 src/transformers/models/phi3_5/processing_phi3_v.py diff --git a/src/transformers/models/phi3_5/configuration_phi3_v.py b/src/transformers/models/phi3_5/configuration_phi3_v.py new file mode 100644 index 000000000000..e3affcbfa11f --- /dev/null +++ b/src/transformers/models/phi3_5/configuration_phi3_v.py @@ -0,0 +1,218 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Phi-3-V model configuration""" + + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +PHI3V_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "microsoft/Phi-3-vision-128k-instruct": "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/resolve/main/config.json", + "microsoft/Phi-3.5-vision-instruct": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct/resolve/main/config.json", +} + + +class Phi3VConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Phi3VModel`]. It is used to instantiate a Phi-3 + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the + [microsoft/Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 32064): + Vocabulary size of the Phi-3-V model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Phi3VModel`]. + hidden_size (`int`, *optional*, defaults to 3072): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 8192): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + resid_pdrop (`float`, *optional*, defaults to 0.0): + Dropout probability for mlp outputs. + embd_pdrop (`int`, *optional*, defaults to 0.0): + The dropout ratio for the embeddings. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio after computing the attention scores. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model might ever be used with. + original_max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model was trained with. This is used to determine the size of the + original RoPE embeddings when using long scaling. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon value used for the RMSNorm. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`dict`, *optional*): + The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must + contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be either `su` or `yarn` and + the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size + divided by the number of attention heads divided by 2. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 32000): + The id of the "end-of-sequence" token. + pad_token_id (`int`, *optional*, defaults to 32000): + The id of the padding token. + sliding_window (`int`, *optional*): + Sliding window attention window size. If `None`, no sliding window is applied. + embd_layer (`str`, *optional*, defaults to `"default"`): + The embedding layer to use. Can be either `"default"` or `"image"`. "default" uses the standard embedding for text. + + Example: + + ```python + >>> from transformers import Phi3VModel, Phi3VConfig + + >>> # Initializing a Phi-3-V style configuration + >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-vision-128k-instruct") + + >>> # Initializing a model from the configuration + >>> model = Phi3VModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "phi3_v" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32064, + hidden_size=3072, + intermediate_size=8192, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + resid_pdrop=0.0, + embd_pdrop=0.0, + attention_dropout=0.0, + hidden_act="silu", + max_position_embeddings=4096, + original_max_position_embeddings=4096, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + bos_token_id=1, + eos_token_id=32000, + pad_token_id=32000, + sliding_window=None, + embd_layer: str = "default", + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attention_dropout = attention_dropout + self.hidden_act = hidden_act + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_validation() + self.sliding_window = sliding_window + self.embd_layer = embd_layer + + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3: + raise ValueError( + "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_short_factor = self.rope_scaling.get("short_factor", None) + rope_scaling_long_factor = self.rope_scaling.get("long_factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["su", "yarn"]: + raise ValueError(f"`rope_scaling`'s type field must be one of ['su', 'yarn'], got {rope_scaling_type}") + if not ( + isinstance(rope_scaling_short_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor) + ): + raise ValueError( + f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}" + ) + if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}" + ) + if not ( + isinstance(rope_scaling_long_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor) + ): + raise ValueError( + f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}" + ) + if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" + ) \ No newline at end of file diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_5/modeling_phi3_v.py new file mode 100644 index 000000000000..f08fb1649b5d --- /dev/null +++ b/src/transformers/models/phi3_5/modeling_phi3_v.py @@ -0,0 +1,1942 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" PyTorch Phi-3-V model.""" + +import inspect +import math +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache +from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_phi3_v import Phi3VConfig + +try: + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) +except ImportError: + pass + +import torch +from torch import nn +from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig +from transformers.models.clip.modeling_clip import CLIPAttention +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +MAX_INPUT_ID = int(1e9) + +CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( + attention_dropout=0.0, + dropout=0.0, + hidden_act="quick_gelu", + hidden_size=1024, + image_size=336, + initializer_factor=1.0, + initializer_range=0.02, + intermediate_size=4096, + layer_norm_eps=1e-05, + num_attention_heads=16, + num_channels=3, + num_hidden_layers=24, + patch_size=14, + projection_dim=768 +) + +class CLIPAttentionFA2(CLIPAttention): + """Add flash attention 2 to CLIPAttention. (This is only used in the vision encoder)""" + + def forward(self, + hidden_states, + attention_mask=None, + causal_attention_mask=None, + output_attentions=False, + ): + """Input shape: Batch x Time x Channel""" + + assert attention_mask is None, "CLIPAttentionFA2 does not support attention_mask" + assert causal_attention_mask is None, "CLIPAttentionFA2 does not support causal_attention_mask" + assert output_attentions is False, "CLIPAttentionFA2 does not support output_attentions" + + bsz, tgt_len, embed_dim = hidden_states.size() + query_states = self.q_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + key_states = self.k_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + value_states = self.v_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout_p=self.dropout if self.training else 0.0, + softmax_scale=self.scale, + causal=False, + ).reshape(bsz, tgt_len, embed_dim) + + attn_output = self.out_proj(attn_output) + return attn_output, None + + +class Phi3ImageEmbedding(nn.Module): + """Phi3 Image embedding.""" + + def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None: + super().__init__() + + # n_embed or hidden_size + hidden_size = config.n_embd if hasattr(config, 'n_embd') else config.hidden_size + if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'): + embd_drop = config.embd_pdrop if hasattr(config, 'embd_pdrop') else config.embed_pdrop + self.drop = nn.Dropout(embd_drop) + else: + self.drop = None + + self.wte = wte + + if isinstance(config.img_processor, dict) and config.img_processor.get('name', None) == 'clip_vision_model': + assert 'model_name' in config.img_processor, 'model_name must be provided for CLIPVisionModel' + assert 'image_dim_out' in config.img_processor, 'image_dim_out must be provided for CLIPVisionModel' + assert 'num_img_tokens' in config.img_processor, 'num_img_tokens must be provided for CLIPVisionModel' + assert config.img_processor['model_name'] == 'openai/clip-vit-large-patch14-336' + clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG + self.img_processor = CLIPVisionModel(clip_config) + image_dim_out = config.img_processor['image_dim_out'] + self.num_img_tokens = config.img_processor['num_img_tokens'] + + # FA2 in CLIP + if config._attn_implementation == 'flash_attention_2': + for layer in self.img_processor.vision_model.encoder.layers: + clip_fa2 = CLIPAttentionFA2(clip_config) + del layer.self_attn + layer.self_attn = clip_fa2 + else: + raise NotImplementedError(f'img_processor = {config.img_processor}, not implemented') + + self.image_dim_out = image_dim_out + self.img_sizes = None + + # global_gn and sub_gn for hd transform, serves as line separator + self.use_hd_transform = kwargs.get('use_hd_transform', False) + self.with_learnable_separator = kwargs.get('with_learnable_separator', False) + self.hd_transform_order = kwargs.get('hd_transform_order', 'glb_sub') + # with_hd_transform and with_learnable_separator should have same value + assert self.use_hd_transform == self.with_learnable_separator, 'use_hd_transform and with_learnable_separator should have same value' + if self.with_learnable_separator: + assert self.use_hd_transform, 'learnable separator is only for hd transform' + # 1024 * 4, merge spatial to channel dimension + self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4])) + self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4])) + logger.info(f'learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}') + + projection_cls = kwargs.get('projection_cls', 'linear') + if projection_cls == 'linear': + self.img_projection = nn.Linear(image_dim_out, hidden_size) + elif projection_cls == 'mlp' and self.use_hd_transform: + dim_projection = hidden_size + depth = 2 + layers = [nn.Linear(image_dim_out * 4, dim_projection)] + for _ in range(1, depth): + layers.extend([nn.GELU(), + nn.Linear(dim_projection, dim_projection)]) + self.img_projection = nn.Sequential(*layers) + elif projection_cls == 'mlp': + dim_projection = hidden_size + depth = 2 + layers = [nn.Linear(image_dim_out, dim_projection)] + for _ in range(1, depth): + layers.extend([nn.GELU(), + nn.Linear(dim_projection, dim_projection)]) + self.img_projection = nn.Sequential(*layers) + else: + raise NotImplementedError(f'projection_cls = {projection_cls}, not implemented') + + self.vocab_size = config.vocab_size + self.img_features = None + + if isinstance(config.img_processor, dict): + self.layer_idx = config.img_processor.get('layer_idx', -2) + self.type_feature = config.img_processor.get('type_feature', 'patch') + else: + self.layer_idx = -2 + self.type_feature = 'patch' + + + def set_img_features(self, img_features: torch.FloatTensor) -> None: + self.img_features = img_features + + def set_img_sizes(self, img_sizes: torch.LongTensor) -> None: + self.img_sizes = img_sizes + + def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor: + LAYER_IDX = self.layer_idx + TYPE_FEATURE = self.type_feature + + img_processor_output = self.img_processor(img_embeds, output_hidden_states=True) + img_feature = img_processor_output.hidden_states[LAYER_IDX] + + if TYPE_FEATURE == "patch": + patch_feature = img_feature[:, 1:] + return patch_feature + + raise NotImplementedError + + def forward( + self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, image_sizes=None + ) -> torch.FloatTensor: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + # positions for image tokens + positions = torch.nonzero((input_ids < 0) & (input_ids > -MAX_INPUT_ID), as_tuple=True) + has_image = len(positions[0].tolist()) > 0 + input_ids = input_ids.clamp_min(0).clamp_max(self.vocab_size).detach() + hidden_states = self.wte(input_ids) + + if has_image: + assert self.use_hd_transform + num_images, num_crops, c, h, w = pixel_values.shape + assert c == 3 and h == w == 336 + img_features = self.get_img_features(pixel_values.flatten(0, 1)).reshape( + num_images, num_crops, -1, self.image_dim_out + ) + image_features_proj = self.hd_feature_transform(img_features, image_sizes) + hidden_states = hidden_states.index_put( + positions, image_features_proj, accumulate=False + ) + + if self.drop is not None: + hidden_states = self.drop(hidden_states) + + return hidden_states + + def hd_feature_transform(self, image_features, image_sizes): + """ + image_features: (num_images, num_crops+1, 24*24, 1024) + """ + assert ( + self.hd_transform_order == 'sub_glb' + ), f'hd_transform_order `{self.hd_transform_order}` not implemented' + if isinstance(self.img_projection, nn.Sequential): + target_device = self.img_projection[0].bias.device + target_dtype = self.img_projection[0].bias.dtype + else: # It's a single nn.Linear layer + target_device = self.img_projection.bias.device + target_dtype = self.img_projection.bias.dtype + + global_image_features = image_features[:, 0] # (num_images, 24*24, 1024) + # global feature can be viewed as a special HD case with num_crops 1x1 + global_image_features_hd = self.reshape_hd_patches_2x2merge(global_image_features, 1, 1) + global_image_features_hd_newline = self.add_image_newline(global_image_features_hd) + + all_image_embeddings = [] + # need a for loop to process each image because of different image sizes + # (patch arrangement is different for each image) + for i, img_size in enumerate(image_sizes): + h, w = img_size + h_crop = h // 336 + w_crop = w // 336 + num_crops = h_crop * w_crop + + # NOTE: real num_crops is padded + # (num_crops, 24*24, 1024) + sub_image_features = image_features[i, 1 : 1 + num_crops] + sub_image_features_hd = self.reshape_hd_patches_2x2merge( + sub_image_features, h_crop, w_crop + ) + sub_image_features_hd_newline = self.add_image_newline(sub_image_features_hd) + + # [sub features, separator, global features] + all_image_embeddings.extend( + [ + sub_image_features_hd_newline.squeeze(0), # (h_crop*12*(w_crop*12+1), 4096) + self.glb_GN.squeeze(0), + global_image_features_hd_newline[i], + ] + ) + + image_features_proj = self.img_projection( + torch.cat(all_image_embeddings, dim=0).to(target_device).to(target_dtype) + ) + + return image_features_proj + + def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): + """ + image_features: (num_images*num_crops, 24*24, 1024) + output: (num_images, h_crop*12, w_crop*12, 4096), h_crop*w_crop == num_crops + """ + N, L, C = image_features.shape + assert L == 24 * 24 and C == 1024 and N % (h_crop * w_crop) == 0 + num_images = N // (h_crop * w_crop) + H = int(L**0.5) + image_features_hd = ( + image_features.reshape(N, H, H, C) # N, 24, 24, 1024 + .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024 + .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024 + .reshape(N, -1, 4 * C) # N, 144, 4096 + .reshape( + num_images, h_crop, w_crop, H // 2, H // 2, -1 + ) # n_img, h_crop, w_crop, 12, 12, 4096 + .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096 + .reshape( + num_images, h_crop * H // 2, w_crop * H // 2, 4 * C + ) # n_img, h_crop*12, w_crop*12, 4096 + ) + + # alternative implementation using einops + # from einops import rearrange + # image_features_nhwc = rearrange( + # image_features, + # 'N (H W) c -> N H W c', + # H=H, + # W=H, + # ) + # image_features_2x2merge = rearrange( + # image_features_nhwc, + # 'N (h h_pool) (w w_pool) c -> N h w (h_pool w_pool c)', + # h_pool=2, + # w_pool=2, + # ) + # image_features_hd = rearrange( + # image_features_2x2merge, + # '(n_img h_crop w_crop) h w C -> n_img (h_crop h) (w_crop w) C', + # h_crop=h_crop, + # w_crop=w_crop, + # ) + + return image_features_hd + + def add_image_newline(self, image_features_hd): + """ + image_features_hd: (num_images, h_crop*12, w_crop*12, 4096) + output: (num_images, (h_crop*12) * (w_crop*12+1), 4096) + """ + num_images, h, w, hid_dim = image_features_hd.shape + # add the newline token to the HD image feature patches + newline_embeddings = self.sub_GN.expand(num_images, h, -1, -1) # (n_img, h, 1, hid_dim) + image_features_hd_newline = torch.cat( + [image_features_hd, newline_embeddings], dim=2 + ).reshape(num_images, -1, hid_dim) + return image_features_hd_newline + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-vision-128k-instruct" +_CONFIG_FOR_DOC = "Phi3VConfig" + +PHI3V_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "microsoft/Phi-3-vision-128k-instruct", + # See all Phi-3 models at https://huggingface.co/models?filter=Phi-3 +] + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Phi3 +class Phi3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Phi3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +# Copied from transformers.models.llama.modeling_llama._get_unpad_data +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3, Gemma->Phi3 +class Phi3RotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.register_buffer("inv_freq", None, persistent=False) + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if self.inv_freq is None: + self.inv_freq = 1.0 / ( + self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim) + ) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Phi3SuScaledRotaryEmbedding(Phi3RotaryEmbedding): + def __init__(self, dim, config, device=None): + super().__init__(dim, config.max_position_embeddings, config.rope_theta, device) + + self.short_factor = config.rope_scaling["short_factor"] + self.long_factor = config.rope_scaling["long_factor"] + self.original_max_position_embeddings = config.original_max_position_embeddings + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + seq_len = seq_len or torch.max(position_ids) + 1 + if seq_len > self.original_max_position_embeddings: + ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device) + else: + ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device) + + inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim + self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape) + + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + + scale = self.max_position_embeddings / self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = math.sqrt(1 + math.log(scale) / math.log(self.original_max_position_embeddings)) + + cos = emb.cos() * scaling_factor + sin = emb.sin() * scaling_factor + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Phi3YarnScaledRotaryEmbedding(Phi3RotaryEmbedding): + def __init__(self, dim, config, device=None): + super().__init__(dim, config.max_position_embeddings, config.rope_theta, device) + + self.short_factor = config.rope_scaling["short_factor"] + self.long_factor = config.rope_scaling["long_factor"] + self.original_max_position_embeddings = config.original_max_position_embeddings + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + seq_len = torch.max(position_ids) + 1 + if seq_len > self.original_max_position_embeddings: + ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device) + else: + ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device) + + inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim + self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape) + + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + + scale = self.max_position_embeddings / self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = 0.1 * math.log(scale) + 1.0 + + cos = emb.cos() * scaling_factor + sin = emb.sin() * scaling_factor + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class Phi3MLP(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) + + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + up_states = self.gate_up_proj(hidden_states) + + gate, up_states = up_states.chunk(2, dim=-1) + up_states = up_states * self.activation_fn(gate) + + return self.down_proj(up_states) + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class Phi3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Phi3VConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.original_max_position_embeddings = config.original_max_position_embeddings + self.rope_theta = config.rope_theta + self.rope_scaling = config.rope_scaling + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + op_size = self.num_heads * self.head_dim + 2 * (self.num_key_value_heads * self.head_dim) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.qkv_proj = nn.Linear(self.hidden_size, op_size, bias=False) + self._init_rope() + + def _init_rope(self): + if self.rope_scaling is None: + self.rotary_emb = Phi3RotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling["type"] + if scaling_type == "su": + self.rotary_emb = Phi3SuScaledRotaryEmbedding(self.head_dim, self.config) + elif scaling_type == "yarn": + self.rotary_emb = Phi3YarnScaledRotaryEmbedding(self.head_dim, self.config) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + logger.warning_once("You are not running the flash-attention implementation, expect numerical differences.") + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class Phi3FlashAttention2(Phi3Attention): + """ + Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # Phi3FlashAttention2 attention does not support output_attentions + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library." + ) + raise ValueError("The current flash attention version does not support sliding window attention.") + + output_attentions = False + + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + + # overwrite attention_mask with padding_mask + attention_mask = kwargs.pop("padding_mask") + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=rotary_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_dropout = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. + + if query_states.dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.qkv_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=attn_dropout, + use_sliding_windows=use_sliding_windows, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._flash_attention_forward + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + query_length, + dropout=0.0, + softmax_scale=None, + use_sliding_windows=False, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + use_sliding_windows (`bool`, *optional*): + Whether to activate sliding window attention. + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + if not use_sliding_windows: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + + +# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3 +# TODO @Arthur no longer copied from LLama after static cache +class Phi3SdpaAttention(Phi3Attention): + """ + Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `Phi3Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from Phi3Attention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "Phi3Model is using Phi3SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal=self.is_causal and attention_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +PHI3_ATTENTION_CLASSES = { + "eager": Phi3Attention, + "flash_attention_2": Phi3FlashAttention2, + "sdpa": Phi3SdpaAttention, +} + + +class Phi3DecoderLayer(nn.Module): + def __init__(self, config: Phi3VConfig, layer_idx: int): + super().__init__() + + self.config = config + self.self_attn = PHI3_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) + + self.mlp = Phi3MLP(config) + self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.resid_attn_dropout = nn.Dropout(config.resid_pdrop) + self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop) + self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + """ + Args: + hidden_states (`torch.FloatTensor`): + input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + position_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + attn_outputs, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = residual + self.resid_attn_dropout(attn_outputs) + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + self.resid_mlp_dropout(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +PHI3V_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Phi3VConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.", + PHI3V_START_DOCSTRING, +) +class Phi3VPreTrainedModel(PreTrainedModel): + config_class = Phi3VConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Phi3DecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = False + _supports_cache_class = True + + _version = "0.0.5" + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +PHI3V_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance; + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`]. + See [`Phi3ImageProcessor.__call__`] for details. + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): + The sizes of the images in the batch, being (height, width) for each image. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.", + PHI3V_START_DOCSTRING, +) +class Phi3VModel(Phi3VPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`] + + Args: + config: Phi3Config + """ + + def __init__(self, config: Phi3VConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.embed_dropout = nn.Dropout(config.embd_pdrop) + + self.vision_embed_tokens = None + if isinstance(config.embd_layer, dict): + # vision embedding layer + embedding_config = { + 'embedding_cls': config.embd_layer['embedding_cls'], + **config.embd_layer + } + self.vision_embed_tokens = Phi3ImageEmbedding(config, wte=self.embed_tokens, **embedding_config) + # # set wte the same for vision embedding + # self.vision_embed_tokens.wte.weight = self.embed_tokens.weight + + self.layers = nn.ModuleList( + [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape[:2] + elif inputs_embeds is not None: + batch_size, seq_length = inputs_embeds.shape[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + past_key_values_length = 0 + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + if use_cache: + use_legacy_cache = not isinstance(past_key_values, Cache) + if use_legacy_cache: + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + past_key_values_length = past_key_values.get_usable_length(seq_length) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + if pixel_values is not None and image_sizes is not None: + assert self.vision_embed_tokens is not None, "Vision embedding layer is not defined" + inputs_embeds = self.vision_embed_tokens(input_ids, pixel_values=pixel_values, image_sizes=image_sizes) + else: + inputs_embeds = self.embed_tokens(input_ids) + + if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache: + is_padding_right = attention_mask[:, -1].sum().item() != batch_size + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'" + " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to " + " call `tokenizer.padding_side = 'left'` before tokenizing the input. " + ) + + if self._attn_implementation == "flash_attention_2": + # 2d mask is passed through the layers + attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None + else: + # 4d mask is passed through the layers + attention_mask = _prepare_4d_causal_attention_mask( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window=self.config.sliding_window, + ) + + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + attention_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class Phi3VForCausalLM(Phi3VPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3 + def __init__(self, config): + super().__init__(config) + self.model = Phi3VModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings + def get_input_embeddings(self): + return self.model.embed_tokens + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings + def get_output_embeddings(self): + return self.lm_head + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder + def set_decoder(self, decoder): + self.model = decoder + + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder + def get_decoder(self): + return self.model + + # Ignore copy + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, Phi3ForCausalLM + + >>> model = Phi3ForCausalLM.from_pretrained("microsoft/phi-3-mini-4k-instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-3-mini-4k-instruct") + + >>> prompt = "This is an example script ." + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + 'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum' + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + image_sizes=image_sizes, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + # Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, pixel_values=None, image_sizes=None, **kwargs + ): + # When the first time input length reached long and short factor switching point, enforce re-compute cache + # It will cause downside of slower at this single token position, however, better than current failure. + if past_key_values and self.config.rope_scaling and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1: + past_length = past_key_values.seen_tokens if isinstance(past_key_values, Cache) else past_key_values[0][0].shape[2] + if past_length <= self.config.original_max_position_embeddings: + past_key_values = None + + if past_key_values is not None: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + past_length = past_key_values.seen_tokens + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_sizes": image_sizes, + } + ) + return model_inputs + + @staticmethod + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +@add_start_docstrings( + """ + The [`Phi3VModel`] with a sequence classification head on top (linear layer). + + [`Phi3VForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + PHI3V_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3, LLAMA->PHI3, self.transformer->self.model, transformer_outputs->model_outputs +class Phi3VForSequenceClassification(Phi3VPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = Phi3VModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + model_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + image_sizes=image_sizes, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = model_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility + sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 + sequence_lengths = sequence_lengths % input_ids.shape[-1] + sequence_lengths = sequence_lengths.to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + model_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=model_outputs.past_key_values, + hidden_states=model_outputs.hidden_states, + attentions=model_outputs.attentions, + ) + + +@add_start_docstrings( + """ + [`Phi3VModel`] with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for + Named-Entity-Recognition (NER) tasks. + """, + PHI3V_START_DOCSTRING, +) +# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3,MPT->PHI3,self.transformer->self.model,transformer_outputs->model_outputs +class Phi3VForTokenClassification(Phi3VPreTrainedModel): + def __init__(self, config: Phi3VConfig): + super().__init__(config) + self.num_labels = config.num_labels + + self.model = Phi3VModel(config) + if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None: + classifier_dropout = config.classifier_dropout + elif hasattr(config, "hidden_dropout") and config.hidden_dropout is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + model_outputs = self.model( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + image_sizes=image_sizes, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = model_outputs[0] + hidden_states = self.dropout(hidden_states) + logits = self.classifier(hidden_states) + + loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(logits.device) + batch_size, seq_length = labels.shape + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length) + ) + + if not return_dict: + output = (logits,) + model_outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=model_outputs.hidden_states, + attentions=model_outputs.attentions, + ) diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py new file mode 100644 index 000000000000..4a4109dd1c20 --- /dev/null +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -0,0 +1,478 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Processor class for Phi3-V. +""" +import re +from typing import List, Optional, Union + +import torch + +import transformers +from transformers.feature_extraction_utils import BatchFeature +from transformers.image_utils import ImageInput +from transformers.processing_utils import ProcessorMixin +from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy +from transformers.utils import TensorType + + +"""Image processor class for Phi3-V.""" + +from typing import List, Optional, Union + +import numpy as np + +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature +from transformers.image_transforms import ( + convert_to_rgb, +) +from transformers.image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ImageInput, + make_list_of_images, + valid_images, +) +from transformers.utils import TensorType, is_vision_available, logging + +from transformers import AutoImageProcessor + +logger = logging.get_logger(__name__) + + +if is_vision_available(): + from PIL import Image + +import torch +import torchvision + +def padding_336(b): + width, height = b.size + tar = int(np.ceil(height / 336) * 336) + top_padding = int((tar - height)/2) + bottom_padding = tar - height - top_padding + left_padding = 0 + right_padding = 0 + b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255]) + + return b + +def calc_padded_size(width, height, padding_unit=336): + target_height = int(np.ceil(height / padding_unit) * padding_unit) + top_padding = int((target_height - height) / 2) + bottom_padding = target_height - height - top_padding + left_padding = 0 + right_padding = 0 + padded_width = width + left_padding + right_padding + padded_height = height + top_padding + bottom_padding + return padded_width, padded_height + +def HD_transform(img, hd_num=16): + width, height = img.size + trans = False + if width < height: + img = img.transpose(Image.TRANSPOSE) + trans = True + width, height = img.size + ratio = (width/ height) + scale = 1 + while scale*np.ceil(scale/ratio) <= hd_num: + scale += 1 + scale -= 1 + new_w = int(scale * 336) + new_h = int(new_w / ratio) + + img = torchvision.transforms.functional.resize(img, [new_h, new_w],) + img = padding_336(img) + width, height = img.size + if trans: + img = img.transpose(Image.TRANSPOSE) + + return img + +def calc_hd_transform_size(width, height, hd_num=16): + transposed = False + if width < height: + width, height = height, width + transposed = True + + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= hd_num: + scale += 1 + scale -= 1 + + new_width = int(scale * 336) + new_height = int(new_width / ratio) + + padded_width, padded_height = calc_padded_size(new_width, new_height) + + if transposed: + padded_width, padded_height = padded_height, padded_width + + return padded_width, padded_height + +def pad_to_max_num_crops_tensor(images, max_crops=5): + """ + images: B x 3 x H x W, B<=max_crops + """ + B, _, H, W = images.shape + if B < max_crops: + pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device) + images = torch.cat([images, pad], dim=0) + return images + + +class Phi3VImageProcessor(BaseImageProcessor): + r""" + Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques + for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512) + + Args: + image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + num_crops: int = 1, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = True, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.num_crops = num_crops + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + self.do_convert_rgb = do_convert_rgb + + def calc_num_image_tokens( + self, + images: ImageInput + ): + """ Calculate the number of image tokens for each image. + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + """ + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + images = [image.convert('RGB') for image in images] + # (H, W, C) + elems = [HD_transform(im, hd_num = self.num_crops) for im in images] + shapes = [[im.size[1], im.size[0]] for im in elems] + num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes] + return num_img_tokens + + def calc_num_image_tokens_from_image_size(self, width, height): + """ + Calculate the number of image tokens for a given image size. + Args: + width (`int`): Width of the image. + height (`int`): Height of the image. + """ + new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops) + num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12) + return num_img_tokens + + def preprocess( + self, + images: ImageInput, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = None, + return_tensors: Optional[Union[str, TensorType]] = None, + ): + """ + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + """ + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + image_sizes = [] + img_processor = torchvision.transforms.Compose([ + torchvision.transforms.ToTensor(), + torchvision.transforms.Normalize(image_mean, image_std) + ]) + + # PIL images + # HD_transform pad images to size of multiiply of 336, 336 + # convert to RGB first + images = [image.convert('RGB') for image in images] + elems = [HD_transform(im, hd_num = self.num_crops) for im in images] + # tensor transform and normalize + hd_images = [img_processor(im) for im in elems] + # create global image + global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images] + + # [(3, h, w)], where h, w is multiple of 336 + shapes = [[im.size(1), im.size(2)] for im in hd_images] + num_img_tokens = [int(((h//336)*(w//336)+1)*144 + 1 + (h//336+1)*12) for h, w in shapes] + # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336) + # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336) + hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)] + # concat global image and local image + hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)] + + # pad to max_num_crops + image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape] + image_transformed = torch.stack(image_transformed, dim=0) + image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes] + padded_images = image_transformed + image_sizes = shapes + + data = {"pixel_values": padded_images, + "image_sizes": image_sizes, + "num_img_tokens": num_img_tokens + } + + return BatchFeature(data=data, tensor_type=return_tensors) + +AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor) + +transformers.Phi3VImageProcessor = Phi3VImageProcessor + +class Phi3VProcessor(ProcessorMixin): + r""" + Constructs a Phi3-V processor which wraps a Phi3-V image processor and a LLaMa tokenizer into a single processor. + + [`Phi3VProcessor`] offers all the functionalities of [`Phi3VImageProcessor`] and [`LlamaTokenizerFast`]. See the + [`~Phi3VProcessor.__call__`] and [`~Phi3VProcessor.decode`] for more information. + + Args: + image_processor ([`Phi3VImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`LlamaTokenizerFast`], *optional*): + The tokenizer is a required input. + """ + + attributes = ["image_processor", "tokenizer"] + image_processor_class = "Phi3VImageProcessor" + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + special_image_token = "<|image|>" + + def __init__(self, image_processor, tokenizer): + self.image_processor = image_processor + self.tokenizer = tokenizer + self.num_img_tokens = image_processor.num_img_tokens + self.img_tokens = [f"<|image_{i+1}|>" for i in range(1000000)] + + def __call__( + self, + text: Union[TextInput, List[TextInput]], + images: ImageInput = None, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TruncationStrategy] = None, + max_length=None, + return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH, + ) -> BatchFeature: + """ + Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` + and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode + the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to + Phi3ImageProcessor's [`~Phi3ImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring + of the above two methods for more information. + + Args: + text (`str`, `List[str]`, `List[List[str]]`): + The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings + (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set + `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch + tensor. Both channels-first and channels-last formats are supported. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + truncation (`bool`, *optional*): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. Acceptable values are: + + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + - `'jax'`: Return JAX `jnp.ndarray` objects. + + Returns: + [`BatchFeature`]: A [`BatchFeature`] with the following fields: + + - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. + - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when + `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not + `None`). + - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. + """ + if images is not None: + image_inputs = self.image_processor(images, return_tensors=return_tensors) + else: + image_inputs = {} + inputs = self._convert_images_texts_to_inputs(image_inputs, text, padding=padding, truncation=truncation, max_length=max_length, return_tensors=return_tensors) + return inputs + + def calc_num_image_tokens(self, images: ImageInput): + """ Calculate the number of image tokens for each image. + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + """ + return self.image_processor.calc_num_image_tokens(images) + + def calc_num_image_tokens_from_image_size(self, width, height): + """ Calculate the number of image token for an image with given width and height. + Args: + width (`int`): + Width of the image. + height (`int`): + Height of the image. + """ + return self.image_processor.calc_num_image_tokens_from_image_size(width, height) + + + @property + def special_image_token_id(self): + return self.tokenizer.convert_tokens_to_ids(self.special_image_token) + + def get_special_image_token_id(self): + return self.tokenizer.convert_tokens_to_ids(self.special_image_token) + + def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None): + + if not len(images): + model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length) + return BatchFeature(data={**model_inputs}) + + pattern = r"<\|image_\d+\|>" + prompt_chunks = [self.tokenizer(chunk).input_ids for chunk in re.split(pattern, texts)] + + if 'num_img_tokens' in images: + num_img_tokens = images['num_img_tokens'] + else: + assert 'num_crops' in images, 'num_crops must be provided in images if num_img_tokens is not provided' + num_crops = images['num_crops'] + num_img_tokens = [_num_crops * self.num_img_tokens for _num_crops in num_crops] + + images, image_sizes = images['pixel_values'], images['image_sizes'] + + # image_tags needs to start from 1 to n + image_tags = re.findall(pattern, texts) + # image_ids = [int(s.split("|")[1].split("_")[-1]) * -1 for s in image_tags] + # image_ids_pad = [[iid]*num_img_tokens[i] for i, iid in enumerate(image_ids)] + image_ids = [int(s.split("|")[1].split("_")[-1]) for s in image_tags] + unique_image_ids = sorted(list(set(image_ids))) + # image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be [1, 4, 5] + # check the condition + assert unique_image_ids == list(range(1, len(unique_image_ids)+1)), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}" + # total images must be the same as the number of image tags + assert len(unique_image_ids) == len(images), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images" + + image_ids_pad = [[-iid]*num_img_tokens[iid-1] for iid in image_ids] + + def insert_separator(X, sep_list): + if len(X) > len(sep_list): + sep_list.append([]) + return [ele for sublist in zip(X, sep_list) for ele in sublist] + input_ids = [] + offset = 0 + for x in insert_separator(prompt_chunks, image_ids_pad): + input_ids.extend(x[offset:]) + + input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) + attention_mask = (input_ids > -1000000).to(torch.long) + + return BatchFeature(data={"input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": images, + "image_sizes": image_sizes}) + + + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) \ No newline at end of file From 6dc60404603a3241beae0a6acdc5309aa3ce5eed Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 4 Feb 2025 15:15:44 -0700 Subject: [PATCH 02/48] added boilerplate code for phi3V --- docs/source/en/_toctree.yml | 2 ++ src/transformers/models/__init__.py | 1 + src/transformers/models/auto/configuration_auto.py | 2 ++ src/transformers/models/auto/modeling_auto.py | 4 ++++ src/transformers/models/phi3_5/__init__.py | 3 +++ 5 files changed, 12 insertions(+) create mode 100644 src/transformers/models/phi3_5/__init__.py diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index d4cfe1711a5e..1d1a4964da63 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -1091,6 +1091,8 @@ title: Perceiver - local: model_doc/perception_lm title: PerceptionLM + - local: model_doc/phi3_5 + title: Phi-3.5-vision-instruct - local: model_doc/phi4_multimodal title: Phi4 Multimodal - local: model_doc/pix2struct diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index f2d7310d3930..d80703c17c9b 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -253,6 +253,7 @@ from .persimmon import * from .phi import * from .phi3 import * + from .phi3_5 import * from .phi4_multimodal import * from .phimoe import * from .phobert import * diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 16d994f4004f..99dd22ac2482 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -295,6 +295,7 @@ ("persimmon", "PersimmonConfig"), ("phi", "PhiConfig"), ("phi3", "Phi3Config"), + ("phi3_5", "Phi3VConfig"), ("phi4_multimodal", "Phi4MultimodalConfig"), ("phimoe", "PhimoeConfig"), ("pix2struct", "Pix2StructConfig"), @@ -724,6 +725,7 @@ ("persimmon", "Persimmon"), ("phi", "Phi"), ("phi3", "Phi3"), + ("phi3_5", "Phi3V"), ("phi4_multimodal", "Phi4Multimodal"), ("phimoe", "Phimoe"), ("phobert", "PhoBERT"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 8041d5db473b..c2fd4c3f8ceb 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -294,6 +294,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonModel"), ("phi", "PhiModel"), ("phi3", "Phi3Model"), + ("phi3_5", "Phi3VModel"), ("phi4_multimodal", "Phi4MultimodalModel"), ("phimoe", "PhimoeModel"), ("pixtral", "PixtralVisionModel"), @@ -694,6 +695,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForCausalLM"), ("phi", "PhiForCausalLM"), ("phi3", "Phi3ForCausalLM"), + ("phi3_5", "Phi3VForCausalLM"), ("phi4_multimodal", "Phi4MultimodalForCausalLM"), ("phimoe", "PhimoeForCausalLM"), ("plbart", "PLBartForCausalLM"), @@ -1239,6 +1241,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForSequenceClassification"), ("phi", "PhiForSequenceClassification"), ("phi3", "Phi3ForSequenceClassification"), + ("phi3_5", "Phi3VForSequenceClassification"), ("phimoe", "PhimoeForSequenceClassification"), ("plbart", "PLBartForSequenceClassification"), ("qdqbert", "QDQBertForSequenceClassification"), @@ -1440,6 +1443,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForTokenClassification"), ("phi", "PhiForTokenClassification"), ("phi3", "Phi3ForTokenClassification"), + ("phi3_5", "Phi3VForTokenClassification"), ("qdqbert", "QDQBertForTokenClassification"), ("qwen2", "Qwen2ForTokenClassification"), ("qwen2_moe", "Qwen2MoeForTokenClassification"), diff --git a/src/transformers/models/phi3_5/__init__.py b/src/transformers/models/phi3_5/__init__.py new file mode 100644 index 000000000000..b6a58690d491 --- /dev/null +++ b/src/transformers/models/phi3_5/__init__.py @@ -0,0 +1,3 @@ +from .configuration_phi3_v import * +from .modeling_phi3_v import * +from .processing_phi3_v import * From cf018a5780cd195f1b31c9b522e09c5040e2d7b8 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Fri, 7 Feb 2025 15:10:00 -0700 Subject: [PATCH 03/48] CI fixes --- docs/source/en/model_doc/phi3_5.md | 50 ++++ .../models/auto/image_processing_auto.py | 1 + .../models/phi3_5/configuration_phi3_v.py | 12 +- .../models/phi3_5/modeling_phi3_v.py | 168 ++++++------ .../models/phi3_5/processing_phi3_v.py | 254 ++++++++++-------- 5 files changed, 279 insertions(+), 206 deletions(-) create mode 100644 docs/source/en/model_doc/phi3_5.md diff --git a/docs/source/en/model_doc/phi3_5.md b/docs/source/en/model_doc/phi3_5.md new file mode 100644 index 000000000000..a506949e64ef --- /dev/null +++ b/docs/source/en/model_doc/phi3_5.md @@ -0,0 +1,50 @@ + + +# Phi-3.5-vision-instruct + + + +## Phi3VConfig + +[[autodoc]] Phi3VConfig + + + + +## Phi3VModel + +[[autodoc]] Phi3VModel + - forward + +## Phi3VForCausalLM + +[[autodoc]] Phi3VForCausalLM + - forward + - generate + +## Phi3VForSequenceClassification + +[[autodoc]] Phi3VForSequenceClassification + - forward + +## Phi3VForTokenClassification + +[[autodoc]] Phi3VForTokenClassification + - forward + + + diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 08c85e61f733..c10e8d7b75a3 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -146,6 +146,7 @@ ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")), ("perceiver", ("PerceiverImageProcessor", "PerceiverImageProcessorFast")), ("perception_lm", (None, "PerceptionLMImageProcessorFast")), + ("phi3_5", ("Phi3VImageProcessor",)), ("phi4_multimodal", (None, "Phi4MultimodalImageProcessorFast")), ("pix2struct", ("Pix2StructImageProcessor", None)), ("pixtral", ("PixtralImageProcessor", "PixtralImageProcessorFast")), diff --git a/src/transformers/models/phi3_5/configuration_phi3_v.py b/src/transformers/models/phi3_5/configuration_phi3_v.py index e3affcbfa11f..92ac0ca725c1 100644 --- a/src/transformers/models/phi3_5/configuration_phi3_v.py +++ b/src/transformers/models/phi3_5/configuration_phi3_v.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Phi-3-V model configuration""" +"""Phi-3-V model configuration""" - -from transformers.configuration_utils import PretrainedConfig -from transformers.utils import logging +from ...configuration_utils import PretrainedConfig +from ...utils import logging logger = logging.get_logger(__name__) @@ -96,7 +95,7 @@ class Phi3VConfig(PretrainedConfig): sliding_window (`int`, *optional*): Sliding window attention window size. If `None`, no sliding window is applied. embd_layer (`str`, *optional*, defaults to `"default"`): - The embedding layer to use. Can be either `"default"` or `"image"`. "default" uses the standard embedding for text. + The embedding layer to use. Can be either `"default"` or `"image"`. "default" uses the standard embedding for text. Example: @@ -168,7 +167,6 @@ def __init__( self.sliding_window = sliding_window self.embd_layer = embd_layer - super().__init__( bos_token_id=bos_token_id, eos_token_id=eos_token_id, @@ -215,4 +213,4 @@ def _rope_scaling_validation(self): if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2: raise ValueError( f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" - ) \ No newline at end of file + ) diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_5/modeling_phi3_v.py index f08fb1649b5d..6bcfd0f79855 100644 --- a/src/transformers/models/phi3_5/modeling_phi3_v.py +++ b/src/transformers/models/phi3_5/modeling_phi3_v.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" PyTorch Phi-3-V model.""" +"""PyTorch Phi-3-V model.""" import inspect import math @@ -26,17 +26,17 @@ from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss -from transformers.activations import ACT2FN -from transformers.cache_utils import Cache, DynamicCache -from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask -from transformers.modeling_outputs import ( +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask +from ...modeling_outputs import ( BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast, TokenClassifierOutput, ) -from transformers.modeling_utils import PreTrainedModel -from transformers.utils import ( +from ...modeling_utils import PreTrainedModel +from ...utils import ( add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, @@ -46,6 +46,7 @@ ) from .configuration_phi3_v import Phi3VConfig + try: from flash_attn import flash_attn_func, flash_attn_varlen_func from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa @@ -55,10 +56,10 @@ pass import torch -from torch import nn -from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig -from transformers.models.clip.modeling_clip import CLIPAttention -from transformers.utils import logging + +from ... import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig +from ...models.clip.modeling_clip import CLIPAttention + logger = logging.get_logger(__name__) @@ -66,26 +67,28 @@ MAX_INPUT_ID = int(1e9) CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( - attention_dropout=0.0, - dropout=0.0, - hidden_act="quick_gelu", - hidden_size=1024, - image_size=336, - initializer_factor=1.0, - initializer_range=0.02, - intermediate_size=4096, - layer_norm_eps=1e-05, - num_attention_heads=16, - num_channels=3, - num_hidden_layers=24, - patch_size=14, - projection_dim=768 + attention_dropout=0.0, + dropout=0.0, + hidden_act="quick_gelu", + hidden_size=1024, + image_size=336, + initializer_factor=1.0, + initializer_range=0.02, + intermediate_size=4096, + layer_norm_eps=1e-05, + num_attention_heads=16, + num_channels=3, + num_hidden_layers=24, + patch_size=14, + projection_dim=768, ) + class CLIPAttentionFA2(CLIPAttention): """Add flash attention 2 to CLIPAttention. (This is only used in the vision encoder)""" - def forward(self, + def forward( + self, hidden_states, attention_mask=None, causal_attention_mask=None, @@ -122,82 +125,83 @@ def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None: super().__init__() # n_embed or hidden_size - hidden_size = config.n_embd if hasattr(config, 'n_embd') else config.hidden_size - if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'): - embd_drop = config.embd_pdrop if hasattr(config, 'embd_pdrop') else config.embed_pdrop + hidden_size = config.n_embd if hasattr(config, "n_embd") else config.hidden_size + if hasattr(config, "embd_pdrop") or hasattr(config, "embed_pdrop"): + embd_drop = config.embd_pdrop if hasattr(config, "embd_pdrop") else config.embed_pdrop self.drop = nn.Dropout(embd_drop) else: self.drop = None self.wte = wte - if isinstance(config.img_processor, dict) and config.img_processor.get('name', None) == 'clip_vision_model': - assert 'model_name' in config.img_processor, 'model_name must be provided for CLIPVisionModel' - assert 'image_dim_out' in config.img_processor, 'image_dim_out must be provided for CLIPVisionModel' - assert 'num_img_tokens' in config.img_processor, 'num_img_tokens must be provided for CLIPVisionModel' - assert config.img_processor['model_name'] == 'openai/clip-vit-large-patch14-336' + if isinstance(config.img_processor, dict) and config.img_processor.get("name", None) == "clip_vision_model": + assert "model_name" in config.img_processor, "model_name must be provided for CLIPVisionModel" + assert "image_dim_out" in config.img_processor, "image_dim_out must be provided for CLIPVisionModel" + assert "num_img_tokens" in config.img_processor, "num_img_tokens must be provided for CLIPVisionModel" + assert config.img_processor["model_name"] == "openai/clip-vit-large-patch14-336" clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG self.img_processor = CLIPVisionModel(clip_config) - image_dim_out = config.img_processor['image_dim_out'] - self.num_img_tokens = config.img_processor['num_img_tokens'] + image_dim_out = config.img_processor["image_dim_out"] + self.num_img_tokens = config.img_processor["num_img_tokens"] # FA2 in CLIP - if config._attn_implementation == 'flash_attention_2': + if config._attn_implementation == "flash_attention_2": for layer in self.img_processor.vision_model.encoder.layers: clip_fa2 = CLIPAttentionFA2(clip_config) del layer.self_attn layer.self_attn = clip_fa2 else: - raise NotImplementedError(f'img_processor = {config.img_processor}, not implemented') + raise NotImplementedError(f"img_processor = {config.img_processor}, not implemented") self.image_dim_out = image_dim_out self.img_sizes = None # global_gn and sub_gn for hd transform, serves as line separator - self.use_hd_transform = kwargs.get('use_hd_transform', False) - self.with_learnable_separator = kwargs.get('with_learnable_separator', False) - self.hd_transform_order = kwargs.get('hd_transform_order', 'glb_sub') + self.use_hd_transform = kwargs.get("use_hd_transform", False) + self.with_learnable_separator = kwargs.get("with_learnable_separator", False) + self.hd_transform_order = kwargs.get("hd_transform_order", "glb_sub") # with_hd_transform and with_learnable_separator should have same value - assert self.use_hd_transform == self.with_learnable_separator, 'use_hd_transform and with_learnable_separator should have same value' + assert ( + self.use_hd_transform == self.with_learnable_separator + ), "use_hd_transform and with_learnable_separator should have same value" if self.with_learnable_separator: - assert self.use_hd_transform, 'learnable separator is only for hd transform' + assert self.use_hd_transform, "learnable separator is only for hd transform" # 1024 * 4, merge spatial to channel dimension self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4])) self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4])) - logger.info(f'learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}') + logger.info( + f"learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}" + ) - projection_cls = kwargs.get('projection_cls', 'linear') - if projection_cls == 'linear': + projection_cls = kwargs.get("projection_cls", "linear") + if projection_cls == "linear": self.img_projection = nn.Linear(image_dim_out, hidden_size) - elif projection_cls == 'mlp' and self.use_hd_transform: + elif projection_cls == "mlp" and self.use_hd_transform: dim_projection = hidden_size depth = 2 layers = [nn.Linear(image_dim_out * 4, dim_projection)] for _ in range(1, depth): - layers.extend([nn.GELU(), - nn.Linear(dim_projection, dim_projection)]) + layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)]) self.img_projection = nn.Sequential(*layers) - elif projection_cls == 'mlp': + elif projection_cls == "mlp": dim_projection = hidden_size depth = 2 layers = [nn.Linear(image_dim_out, dim_projection)] for _ in range(1, depth): - layers.extend([nn.GELU(), - nn.Linear(dim_projection, dim_projection)]) + layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)]) self.img_projection = nn.Sequential(*layers) else: - raise NotImplementedError(f'projection_cls = {projection_cls}, not implemented') + raise NotImplementedError(f"projection_cls = {projection_cls}, not implemented") self.vocab_size = config.vocab_size self.img_features = None if isinstance(config.img_processor, dict): - self.layer_idx = config.img_processor.get('layer_idx', -2) - self.type_feature = config.img_processor.get('type_feature', 'patch') + self.layer_idx = config.img_processor.get("layer_idx", -2) + self.type_feature = config.img_processor.get("type_feature", "patch") else: self.layer_idx = -2 - self.type_feature = 'patch' - + self.type_feature = "patch" def set_img_features(self, img_features: torch.FloatTensor) -> None: self.img_features = img_features @@ -238,9 +242,7 @@ def forward( num_images, num_crops, -1, self.image_dim_out ) image_features_proj = self.hd_feature_transform(img_features, image_sizes) - hidden_states = hidden_states.index_put( - positions, image_features_proj, accumulate=False - ) + hidden_states = hidden_states.index_put(positions, image_features_proj, accumulate=False) if self.drop is not None: hidden_states = self.drop(hidden_states) @@ -251,9 +253,7 @@ def hd_feature_transform(self, image_features, image_sizes): """ image_features: (num_images, num_crops+1, 24*24, 1024) """ - assert ( - self.hd_transform_order == 'sub_glb' - ), f'hd_transform_order `{self.hd_transform_order}` not implemented' + assert self.hd_transform_order == "sub_glb", f"hd_transform_order `{self.hd_transform_order}` not implemented" if isinstance(self.img_projection, nn.Sequential): target_device = self.img_projection[0].bias.device target_dtype = self.img_projection[0].bias.dtype @@ -278,9 +278,7 @@ def hd_feature_transform(self, image_features, image_sizes): # NOTE: real num_crops is padded # (num_crops, 24*24, 1024) sub_image_features = image_features[i, 1 : 1 + num_crops] - sub_image_features_hd = self.reshape_hd_patches_2x2merge( - sub_image_features, h_crop, w_crop - ) + sub_image_features_hd = self.reshape_hd_patches_2x2merge(sub_image_features, h_crop, w_crop) sub_image_features_hd_newline = self.add_image_newline(sub_image_features_hd) # [sub features, separator, global features] @@ -312,13 +310,9 @@ def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024 .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024 .reshape(N, -1, 4 * C) # N, 144, 4096 - .reshape( - num_images, h_crop, w_crop, H // 2, H // 2, -1 - ) # n_img, h_crop, w_crop, 12, 12, 4096 + .reshape(num_images, h_crop, w_crop, H // 2, H // 2, -1) # n_img, h_crop, w_crop, 12, 12, 4096 .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096 - .reshape( - num_images, h_crop * H // 2, w_crop * H // 2, 4 * C - ) # n_img, h_crop*12, w_crop*12, 4096 + .reshape(num_images, h_crop * H // 2, w_crop * H // 2, 4 * C) # n_img, h_crop*12, w_crop*12, 4096 ) # alternative implementation using einops @@ -352,9 +346,9 @@ def add_image_newline(self, image_features_hd): num_images, h, w, hid_dim = image_features_hd.shape # add the newline token to the HD image feature patches newline_embeddings = self.sub_GN.expand(num_images, h, -1, -1) # (n_img, h, 1, hid_dim) - image_features_hd_newline = torch.cat( - [image_features_hd, newline_embeddings], dim=2 - ).reshape(num_images, -1, hid_dim) + image_features_hd_newline = torch.cat([image_features_hd, newline_embeddings], dim=2).reshape( + num_images, -1, hid_dim + ) return image_features_hd_newline @@ -1344,10 +1338,7 @@ def __init__(self, config: Phi3VConfig): self.vision_embed_tokens = None if isinstance(config.embd_layer, dict): # vision embedding layer - embedding_config = { - 'embedding_cls': config.embd_layer['embedding_cls'], - **config.embd_layer - } + embedding_config = {"embedding_cls": config.embd_layer["embedding_cls"], **config.embd_layer} self.vision_embed_tokens = Phi3ImageEmbedding(config, wte=self.embed_tokens, **embedding_config) # # set wte the same for vision embedding # self.vision_embed_tokens.wte.weight = self.embed_tokens.weight @@ -1645,12 +1636,25 @@ def forward( # Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, pixel_values=None, image_sizes=None, **kwargs + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + pixel_values=None, + image_sizes=None, + **kwargs, ): # When the first time input length reached long and short factor switching point, enforce re-compute cache # It will cause downside of slower at this single token position, however, better than current failure. - if past_key_values and self.config.rope_scaling and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1: - past_length = past_key_values.seen_tokens if isinstance(past_key_values, Cache) else past_key_values[0][0].shape[2] + if ( + past_key_values + and self.config.rope_scaling + and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 + ): + past_length = ( + past_key_values.seen_tokens if isinstance(past_key_values, Cache) else past_key_values[0][0].shape[2] + ) if past_length <= self.config.original_max_position_embeddings: past_key_values = None diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index 4a4109dd1c20..b942d4b04070 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -16,69 +16,62 @@ """ Processor class for Phi3-V. """ -import re -from typing import List, Optional, Union - -import torch - -import transformers -from transformers.feature_extraction_utils import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessorMixin -from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy -from transformers.utils import TensorType - - -"""Image processor class for Phi3-V.""" +import re from typing import List, Optional, Union import numpy as np +import torch -from transformers.image_processing_utils import BaseImageProcessor, BatchFeature -from transformers.image_transforms import ( +from ...image_processing_utils import BaseImageProcessor, BatchFeature +from ...image_transforms import ( convert_to_rgb, ) -from transformers.image_utils import ( +from ...image_utils import ( OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, make_list_of_images, valid_images, ) -from transformers.utils import TensorType, is_vision_available, logging - -from transformers import AutoImageProcessor - -logger = logging.get_logger(__name__) +from ...processing_utils import ProcessorMixin +from ...tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy +from ...utils import TensorType, is_vision_available, logging if is_vision_available(): from PIL import Image -import torch import torchvision + +logger = logging.get_logger(__name__) + + def padding_336(b): width, height = b.size tar = int(np.ceil(height / 336) * 336) - top_padding = int((tar - height)/2) + top_padding = int((tar - height) / 2) bottom_padding = tar - height - top_padding left_padding = 0 right_padding = 0 - b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255]) + b = torchvision.transforms.functional.pad( + b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255, 255, 255] + ) return b -def calc_padded_size(width, height, padding_unit=336): - target_height = int(np.ceil(height / padding_unit) * padding_unit) - top_padding = int((target_height - height) / 2) - bottom_padding = target_height - height - top_padding - left_padding = 0 - right_padding = 0 - padded_width = width + left_padding + right_padding - padded_height = height + top_padding + bottom_padding - return padded_width, padded_height + +def calc_padded_size(width, height, padding_unit=336): + target_height = int(np.ceil(height / padding_unit) * padding_unit) + top_padding = int((target_height - height) / 2) + bottom_padding = target_height - height - top_padding + left_padding = 0 + right_padding = 0 + padded_width = width + left_padding + right_padding + padded_height = height + top_padding + bottom_padding + return padded_width, padded_height + def HD_transform(img, hd_num=16): width, height = img.size @@ -87,15 +80,18 @@ def HD_transform(img, hd_num=16): img = img.transpose(Image.TRANSPOSE) trans = True width, height = img.size - ratio = (width/ height) + ratio = width / height scale = 1 - while scale*np.ceil(scale/ratio) <= hd_num: + while scale * np.ceil(scale / ratio) <= hd_num: scale += 1 scale -= 1 new_w = int(scale * 336) new_h = int(new_w / ratio) - img = torchvision.transforms.functional.resize(img, [new_h, new_w],) + img = torchvision.transforms.functional.resize( + img, + [new_h, new_w], + ) img = padding_336(img) width, height = img.size if trans: @@ -103,27 +99,29 @@ def HD_transform(img, hd_num=16): return img -def calc_hd_transform_size(width, height, hd_num=16): - transposed = False - if width < height: - width, height = height, width - transposed = True - - ratio = width / height - scale = 1 - while scale * np.ceil(scale / ratio) <= hd_num: - scale += 1 - scale -= 1 - - new_width = int(scale * 336) - new_height = int(new_width / ratio) - - padded_width, padded_height = calc_padded_size(new_width, new_height) - - if transposed: - padded_width, padded_height = padded_height, padded_width - - return padded_width, padded_height + +def calc_hd_transform_size(width, height, hd_num=16): + transposed = False + if width < height: + width, height = height, width + transposed = True + + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= hd_num: + scale += 1 + scale -= 1 + + new_width = int(scale * 336) + new_height = int(new_width / ratio) + + padded_width, padded_height = calc_padded_size(new_width, new_height) + + if transposed: + padded_width, padded_height = padded_height, padded_width + + return padded_width, padded_height + def pad_to_max_num_crops_tensor(images, max_crops=5): """ @@ -168,12 +166,9 @@ def __init__( self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD self.do_convert_rgb = do_convert_rgb - - def calc_num_image_tokens( - self, - images: ImageInput - ): - """ Calculate the number of image tokens for each image. + + def calc_num_image_tokens(self, images: ImageInput): + """Calculate the number of image tokens for each image. Args: images (`ImageInput`): Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If @@ -187,11 +182,11 @@ def calc_num_image_tokens( "torch.Tensor, tf.Tensor or jax.ndarray." ) - images = [image.convert('RGB') for image in images] + images = [image.convert("RGB") for image in images] # (H, W, C) - elems = [HD_transform(im, hd_num = self.num_crops) for im in images] + elems = [HD_transform(im, hd_num=self.num_crops) for im in images] shapes = [[im.size[1], im.size[0]] for im in elems] - num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes] + num_img_tokens = [int((h // 336 * w // 336 + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] return num_img_tokens def calc_num_image_tokens_from_image_size(self, width, height): @@ -201,8 +196,8 @@ def calc_num_image_tokens_from_image_size(self, width, height): width (`int`): Width of the image. height (`int`): Height of the image. """ - new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops) - num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12) + new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops) + num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12) return num_img_tokens def preprocess( @@ -249,47 +244,55 @@ def preprocess( images = [convert_to_rgb(image) for image in images] image_sizes = [] - img_processor = torchvision.transforms.Compose([ - torchvision.transforms.ToTensor(), - torchvision.transforms.Normalize(image_mean, image_std) - ]) + img_processor = torchvision.transforms.Compose( + [torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(image_mean, image_std)] + ) # PIL images # HD_transform pad images to size of multiiply of 336, 336 # convert to RGB first - images = [image.convert('RGB') for image in images] - elems = [HD_transform(im, hd_num = self.num_crops) for im in images] + images = [image.convert("RGB") for image in images] + elems = [HD_transform(im, hd_num=self.num_crops) for im in images] # tensor transform and normalize hd_images = [img_processor(im) for im in elems] - # create global image - global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images] + # create global image + global_image = [ + torch.nn.functional.interpolate( + im.unsqueeze(0).float(), + size=(336, 336), + mode="bicubic", + ).to(im.dtype) + for im in hd_images + ] # [(3, h, w)], where h, w is multiple of 336 shapes = [[im.size(1), im.size(2)] for im in hd_images] - num_img_tokens = [int(((h//336)*(w//336)+1)*144 + 1 + (h//336+1)*12) for h, w in shapes] + num_img_tokens = [int(((h // 336) * (w // 336) + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336) # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336) - hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)] + hd_images_reshape = [ + im.reshape(1, 3, h // 336, 336, w // 336, 336) + .permute(0, 2, 4, 1, 3, 5) + .reshape(-1, 3, 336, 336) + .contiguous() + for im, (h, w) in zip(hd_images, shapes) + ] # concat global image and local image - hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)] + hd_images_reshape = [ + torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape) + ] # pad to max_num_crops - image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape] + image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops + 1) for im in hd_images_reshape] image_transformed = torch.stack(image_transformed, dim=0) image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes] padded_images = image_transformed image_sizes = shapes - data = {"pixel_values": padded_images, - "image_sizes": image_sizes, - "num_img_tokens": num_img_tokens - } + data = {"pixel_values": padded_images, "image_sizes": image_sizes, "num_img_tokens": num_img_tokens} return BatchFeature(data=data, tensor_type=return_tensors) -AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor) - -transformers.Phi3VImageProcessor = Phi3VImageProcessor class Phi3VProcessor(ProcessorMixin): r""" @@ -374,20 +377,27 @@ def __call__( image_inputs = self.image_processor(images, return_tensors=return_tensors) else: image_inputs = {} - inputs = self._convert_images_texts_to_inputs(image_inputs, text, padding=padding, truncation=truncation, max_length=max_length, return_tensors=return_tensors) + inputs = self._convert_images_texts_to_inputs( + image_inputs, + text, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + ) return inputs def calc_num_image_tokens(self, images: ImageInput): - """ Calculate the number of image tokens for each image. + """Calculate the number of image tokens for each image. Args: images (`ImageInput`): Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If passing in images with pixel values between 0 and 1, set `do_rescale=False`. """ return self.image_processor.calc_num_image_tokens(images) - + def calc_num_image_tokens_from_image_size(self, width, height): - """ Calculate the number of image token for an image with given width and height. + """Calculate the number of image token for an image with given width and height. Args: width (`int`): Width of the image. @@ -395,64 +405,74 @@ def calc_num_image_tokens_from_image_size(self, width, height): Height of the image. """ return self.image_processor.calc_num_image_tokens_from_image_size(width, height) - - - @property + + @property def special_image_token_id(self): return self.tokenizer.convert_tokens_to_ids(self.special_image_token) def get_special_image_token_id(self): return self.tokenizer.convert_tokens_to_ids(self.special_image_token) - - def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None): + def _convert_images_texts_to_inputs( + self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None + ): if not len(images): - model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length) + model_inputs = self.tokenizer( + texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length + ) return BatchFeature(data={**model_inputs}) pattern = r"<\|image_\d+\|>" - prompt_chunks = [self.tokenizer(chunk).input_ids for chunk in re.split(pattern, texts)] + prompt_chunks = [self.tokenizer(chunk).input_ids for chunk in re.split(pattern, texts)] - if 'num_img_tokens' in images: - num_img_tokens = images['num_img_tokens'] + if "num_img_tokens" in images: + num_img_tokens = images["num_img_tokens"] else: - assert 'num_crops' in images, 'num_crops must be provided in images if num_img_tokens is not provided' - num_crops = images['num_crops'] - num_img_tokens = [_num_crops * self.num_img_tokens for _num_crops in num_crops] + assert "num_crops" in images, "num_crops must be provided in images if num_img_tokens is not provided" + num_crops = images["num_crops"] + num_img_tokens = [_num_crops * self.num_img_tokens for _num_crops in num_crops] - images, image_sizes = images['pixel_values'], images['image_sizes'] + images, image_sizes = images["pixel_values"], images["image_sizes"] # image_tags needs to start from 1 to n - image_tags = re.findall(pattern, texts) + image_tags = re.findall(pattern, texts) # image_ids = [int(s.split("|")[1].split("_")[-1]) * -1 for s in image_tags] # image_ids_pad = [[iid]*num_img_tokens[i] for i, iid in enumerate(image_ids)] image_ids = [int(s.split("|")[1].split("_")[-1]) for s in image_tags] - unique_image_ids = sorted(list(set(image_ids))) + unique_image_ids = sorted(set(image_ids)) # image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be [1, 4, 5] # check the condition - assert unique_image_ids == list(range(1, len(unique_image_ids)+1)), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}" + assert unique_image_ids == list( + range(1, len(unique_image_ids) + 1) + ), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}" # total images must be the same as the number of image tags - assert len(unique_image_ids) == len(images), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images" + assert ( + len(unique_image_ids) == len(images) + ), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images" - image_ids_pad = [[-iid]*num_img_tokens[iid-1] for iid in image_ids] + image_ids_pad = [[-iid] * num_img_tokens[iid - 1] for iid in image_ids] def insert_separator(X, sep_list): if len(X) > len(sep_list): sep_list.append([]) return [ele for sublist in zip(X, sep_list) for ele in sublist] + input_ids = [] - offset = 0 + offset = 0 for x in insert_separator(prompt_chunks, image_ids_pad): input_ids.extend(x[offset:]) input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) attention_mask = (input_ids > -1000000).to(torch.long) - return BatchFeature(data={"input_ids": input_ids, - "attention_mask": attention_mask, - "pixel_values": images, - "image_sizes": image_sizes}) - + return BatchFeature( + data={ + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": images, + "image_sizes": image_sizes, + } + ) # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama def batch_decode(self, *args, **kwargs): @@ -475,4 +495,4 @@ def decode(self, *args, **kwargs): def model_input_names(self): tokenizer_input_names = self.tokenizer.model_input_names image_processor_input_names = self.image_processor.model_input_names - return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) \ No newline at end of file + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) From 042f875721b6324b3835e7dbbe82d5556c0f91bc Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sat, 8 Feb 2025 00:11:43 -0700 Subject: [PATCH 04/48] CI fixes, round two --- .../models/phi3_5/modeling_phi3_v.py | 109 +++++++++--------- .../models/phi3_5/processing_phi3_v.py | 4 +- 2 files changed, 55 insertions(+), 58 deletions(-) diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_5/modeling_phi3_v.py index 6bcfd0f79855..78dee4acdce5 100644 --- a/src/transformers/models/phi3_5/modeling_phi3_v.py +++ b/src/transformers/models/phi3_5/modeling_phi3_v.py @@ -363,7 +363,6 @@ def add_image_newline(self, image_features_hd): ] -# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Phi3 class Phi3RMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): """ @@ -381,7 +380,6 @@ def forward(self, hidden_states): return self.weight * hidden_states.to(input_dtype) -# Copied from transformers.models.llama.modeling_llama._get_unpad_data def _get_unpad_data(attention_mask): seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() @@ -394,27 +392,53 @@ def _get_unpad_data(attention_mask): ) -# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3, Gemma->Phi3 +# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3_5, Gemma->Phi3V class Phi3RotaryEmbedding(nn.Module): - def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + def __init__(self, config: Phi3VConfig, device=None): super().__init__() + # BC: "rope_type" was originally "type" + if hasattr(config, "rope_scaling") and config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq - self.dim = dim - self.max_position_embeddings = max_position_embeddings - self.base = base - self.register_buffer("inv_freq", None, persistent=False) + def _dynamic_frequency_update(self, position_ids, device): + """ + dynamic RoPE layers should recompute `inv_freq` in the following situations: + 1 - growing beyond the cached sequence length (allow scaling) + 2 - the current sequence length is in the original scale (avoid losing precision with small sequences) + """ + seq_len = torch.max(position_ids) + 1 + if seq_len > self.max_seq_len_cached: # growth + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len) + self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation + self.max_seq_len_cached = seq_len + + if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset + # This .to() is needed if the model has been moved to a device after being initialized (because + # the buffer is automatically moved, but not the original copy) + self.original_inv_freq = self.original_inv_freq.to(device) + self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + self.max_seq_len_cached = self.original_max_seq_len @torch.no_grad() - def forward(self, x, position_ids, seq_len=None): - # x: [bs, num_attention_heads, seq_len, head_size] - if self.inv_freq is None: - self.inv_freq = 1.0 / ( - self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim) - ) + def forward(self, x, position_ids): + if "dynamic" in self.rope_type: + self._dynamic_frequency_update(position_ids, device=x.device) + + # Core RoPE block inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) position_ids_expanded = position_ids[:, None, :].float() - # Force float32 since bfloat16 loses precision on long contexts - # See https://github.com/huggingface/transformers/pull/29285 + # Force float32 (see https://github.com/huggingface/transformers/pull/29285) device_type = x.device.type device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" with torch.autocast(device_type=device_type, enabled=False): @@ -422,6 +446,11 @@ def forward(self, x, position_ids, seq_len=None): emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() sin = emb.sin() + + # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention + cos = cos * self.attention_scaling + sin = sin * self.attention_scaling + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) @@ -717,7 +746,6 @@ class Phi3FlashAttention2(Phi3Attention): flash attention and deal with padding tokens in case the input contains any of them. """ - # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -874,7 +902,6 @@ def forward( return attn_output, attn_weights, past_key_value - # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._flash_attention_forward def _flash_attention_forward( self, query_states, @@ -975,7 +1002,6 @@ def _flash_attention_forward( return attn_output - # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape @@ -1019,7 +1045,7 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query ) -# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3 +# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3V # TODO @Arthur no longer copied from LLama after static cache class Phi3SdpaAttention(Phi3Attention): """ @@ -1324,7 +1350,7 @@ class Phi3VModel(Phi3VPreTrainedModel): Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`] Args: - config: Phi3Config + config: Phi3VConfig """ def __init__(self, config: Phi3VConfig): @@ -1506,7 +1532,7 @@ def forward( class Phi3VForCausalLM(Phi3VPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3 + # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3V def __init__(self, config): super().__init__(config) self.model = Phi3VModel(config) @@ -1634,7 +1660,6 @@ def forward( attentions=outputs.attentions, ) - # Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation def prepare_inputs_for_generation( self, input_ids, @@ -1714,7 +1739,6 @@ def prepare_inputs_for_generation( return model_inputs @staticmethod - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache def _reorder_cache(past_key_values, beam_idx): reordered_past = () for layer_past in past_key_values: @@ -1739,7 +1763,7 @@ def _reorder_cache(past_key_values, beam_idx): """, PHI3V_START_DOCSTRING, ) -# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3, LLAMA->PHI3, self.transformer->self.model, transformer_outputs->model_outputs +# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3V, LLAMA->PHI3V, self.transformer->self.model, transformer_outputs->model_outputs class Phi3VForSequenceClassification(Phi3VPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1759,13 +1783,11 @@ def set_input_embeddings(self, value): @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) def forward( self, - input_ids: torch.LongTensor = None, + input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, - pixel_values: Optional[torch.FloatTensor] = None, - image_sizes: Optional[torch.LongTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None, @@ -1786,8 +1808,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - pixel_values=pixel_values, - image_sizes=image_sizes, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, @@ -1818,27 +1838,8 @@ def forward( loss = None if labels is not None: - labels = labels.to(logits.device) - if self.config.problem_type is None: - if self.num_labels == 1: - self.config.problem_type = "regression" - elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): - self.config.problem_type = "single_label_classification" - else: - self.config.problem_type = "multi_label_classification" - - if self.config.problem_type == "regression": - loss_fct = MSELoss() - if self.num_labels == 1: - loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) - else: - loss = loss_fct(pooled_logits, labels) - elif self.config.problem_type == "single_label_classification": - loss_fct = CrossEntropyLoss() - loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) - elif self.config.problem_type == "multi_label_classification": - loss_fct = BCEWithLogitsLoss() - loss = loss_fct(pooled_logits, labels) + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + if not return_dict: output = (pooled_logits,) + model_outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1859,7 +1860,7 @@ def forward( """, PHI3V_START_DOCSTRING, ) -# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3,MPT->PHI3,self.transformer->self.model,transformer_outputs->model_outputs +# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3V,MPT->PHI3V,self.transformer->self.model,transformer_outputs->model_outputs class Phi3VForTokenClassification(Phi3VPreTrainedModel): def __init__(self, config: Phi3VConfig): super().__init__(config) @@ -1890,8 +1891,6 @@ def forward( past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, attention_mask: Optional[torch.Tensor] = None, inputs_embeds: Optional[torch.Tensor] = None, - pixel_values: Optional[torch.FloatTensor] = None, - image_sizes: Optional[torch.LongTensor] = None, labels: Optional[torch.Tensor] = None, use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None, @@ -1912,8 +1911,6 @@ def forward( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - pixel_values=pixel_values, - image_sizes=image_sizes, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index b942d4b04070..03fab4fb22d1 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -302,9 +302,9 @@ class Phi3VProcessor(ProcessorMixin): [`~Phi3VProcessor.__call__`] and [`~Phi3VProcessor.decode`] for more information. Args: - image_processor ([`Phi3VImageProcessor`], *optional*): + image_processor ([`Phi3VImageProcessor`]): The image processor is a required input. - tokenizer ([`LlamaTokenizerFast`], *optional*): + tokenizer ([`LlamaTokenizerFast`]): The tokenizer is a required input. """ From f6b7643db6deb6a9256414ce4f3677ac1af16105 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 9 Feb 2025 12:58:57 -0700 Subject: [PATCH 05/48] fixed various import issues --- docs/source/en/model_doc/phi3_5.md | 9 ++++ src/transformers/models/phi3_5/__init__.py | 59 ++++++++++++++++++++-- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/docs/source/en/model_doc/phi3_5.md b/docs/source/en/model_doc/phi3_5.md index a506949e64ef..efcb09909656 100644 --- a/docs/source/en/model_doc/phi3_5.md +++ b/docs/source/en/model_doc/phi3_5.md @@ -22,6 +22,15 @@ rendered properly in your Markdown viewer. [[autodoc]] Phi3VConfig +## Phi3VImageProcessor + +[[autodoc]] Phi3VImageProcessor + - preprocess + +## Phi3VProcessor + +[[autodoc]] Phi3VProcessor + diff --git a/src/transformers/models/phi3_5/__init__.py b/src/transformers/models/phi3_5/__init__.py index b6a58690d491..e957ae3ca575 100644 --- a/src/transformers/models/phi3_5/__init__.py +++ b/src/transformers/models/phi3_5/__init__.py @@ -1,3 +1,56 @@ -from .configuration_phi3_v import * -from .modeling_phi3_v import * -from .processing_phi3_v import * +from typing import TYPE_CHECKING + +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, +) + + +_import_structure = { + "configuration_phi3_v": ["Phi3VConfig"], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_phi3_v"] = [ + "Phi3VForCausalLM", + "Phi3VForSequenceClassification", + "Phi3VForTokenClassification", + "Phi3VModel", + "Phi3VPreTrainedModel", + ] + _import_structure["processing_phi3_v"] = ["Phi3VImageProcessor", "Phi3VProcessor"] + + +if TYPE_CHECKING: + from .configuration_phi3_v import Phi3VConfig + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_phi3_v import ( + Phi3VForCausalLM, + Phi3VForSequenceClassification, + Phi3VForTokenClassification, + Phi3VModel, + Phi3VPreTrainedModel, + ) + from .processing_phi3_v import Phi3VImageProcessor, Phi3VProcessor + + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) + +#from .configuration_phi3_v import * +#from .modeling_phi3_v import * +#from .processing_phi3_v import * From 888fed6048a397bcbbb5e386769f458fda311606 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 10 Feb 2025 16:56:03 -0700 Subject: [PATCH 06/48] additional import fixes --- src/transformers/models/phi3_5/processing_phi3_v.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index 03fab4fb22d1..c4049e431093 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -21,12 +21,9 @@ from typing import List, Optional, Union import numpy as np -import torch from ...image_processing_utils import BaseImageProcessor, BatchFeature -from ...image_transforms import ( - convert_to_rgb, -) +from ...image_transforms import convert_to_rgb from ...image_utils import ( OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, @@ -36,13 +33,17 @@ ) from ...processing_utils import ProcessorMixin from ...tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy -from ...utils import TensorType, is_vision_available, logging +from ...utils import TensorType, logging, is_vision_available, is_torch_available, is_torchvision_available if is_vision_available(): from PIL import Image -import torchvision +if is_torch_available(): + import torch + +if is_torchvision_available(): + import torchvision logger = logging.get_logger(__name__) From e50604c52d60f68c957f715b19586f3d14d841dd Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 11 Feb 2025 01:49:15 -0700 Subject: [PATCH 07/48] hopefully the last of the import fixes --- src/transformers/models/phi3_5/__init__.py | 19 +- .../models/phi3_5/image_processing_phi3_v.py | 292 ++++++++++++++++++ .../models/phi3_5/processing_phi3_v.py | 270 +--------------- 3 files changed, 312 insertions(+), 269 deletions(-) create mode 100644 src/transformers/models/phi3_5/image_processing_phi3_v.py diff --git a/src/transformers/models/phi3_5/__init__.py b/src/transformers/models/phi3_5/__init__.py index e957ae3ca575..cdeca0d48e8d 100644 --- a/src/transformers/models/phi3_5/__init__.py +++ b/src/transformers/models/phi3_5/__init__.py @@ -4,6 +4,7 @@ OptionalDependencyNotAvailable, _LazyModule, is_torch_available, + is_torchvision_available ) @@ -24,7 +25,14 @@ "Phi3VModel", "Phi3VPreTrainedModel", ] - _import_structure["processing_phi3_v"] = ["Phi3VImageProcessor", "Phi3VProcessor"] + _import_structure["processing_phi3_v"] = ["Phi3VProcessor"] + try: + if not is_torchvision_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + _import_structure["image_processing_phi3_v"] = ["Phi3VImageProcessor"] if TYPE_CHECKING: @@ -43,7 +51,14 @@ Phi3VModel, Phi3VPreTrainedModel, ) - from .processing_phi3_v import Phi3VImageProcessor, Phi3VProcessor + from .processing_phi3_v import Phi3VProcessor + try: + if not is_torchvision_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .image_processing_phi3_v import Phi3VImageProcessor else: diff --git a/src/transformers/models/phi3_5/image_processing_phi3_v.py b/src/transformers/models/phi3_5/image_processing_phi3_v.py new file mode 100644 index 000000000000..dc06dbe23772 --- /dev/null +++ b/src/transformers/models/phi3_5/image_processing_phi3_v.py @@ -0,0 +1,292 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Image Processor class for Phi3-V. +""" + +from typing import List, Optional, Union + +import numpy as np + +from ...image_processing_utils import BaseImageProcessor, BatchFeature +from ...image_transforms import convert_to_rgb +from ...image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ImageInput, + make_list_of_images, + valid_images, +) +from ...utils import TensorType, logging, is_vision_available, is_torch_available, is_torchvision_available + + +if is_vision_available(): + from PIL import Image + +if is_torch_available(): + import torch + +if is_torchvision_available(): + import torchvision + + +logger = logging.get_logger(__name__) + + +def padding_336(b): + width, height = b.size + tar = int(np.ceil(height / 336) * 336) + top_padding = int((tar - height) / 2) + bottom_padding = tar - height - top_padding + left_padding = 0 + right_padding = 0 + b = torchvision.transforms.functional.pad( + b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255, 255, 255] + ) + + return b + + +def calc_padded_size(width, height, padding_unit=336): + target_height = int(np.ceil(height / padding_unit) * padding_unit) + top_padding = int((target_height - height) / 2) + bottom_padding = target_height - height - top_padding + left_padding = 0 + right_padding = 0 + padded_width = width + left_padding + right_padding + padded_height = height + top_padding + bottom_padding + return padded_width, padded_height + + +def HD_transform(img, hd_num=16): + width, height = img.size + trans = False + if width < height: + img = img.transpose(Image.TRANSPOSE) + trans = True + width, height = img.size + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= hd_num: + scale += 1 + scale -= 1 + new_w = int(scale * 336) + new_h = int(new_w / ratio) + + img = torchvision.transforms.functional.resize( + img, + [new_h, new_w], + ) + img = padding_336(img) + width, height = img.size + if trans: + img = img.transpose(Image.TRANSPOSE) + + return img + + +def calc_hd_transform_size(width, height, hd_num=16): + transposed = False + if width < height: + width, height = height, width + transposed = True + + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= hd_num: + scale += 1 + scale -= 1 + + new_width = int(scale * 336) + new_height = int(new_width / ratio) + + padded_width, padded_height = calc_padded_size(new_width, new_height) + + if transposed: + padded_width, padded_height = padded_height, padded_width + + return padded_width, padded_height + + +def pad_to_max_num_crops_tensor(images, max_crops=5): + """ + images: B x 3 x H x W, B<=max_crops + """ + B, _, H, W = images.shape + if B < max_crops: + pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device) + images = torch.cat([images, pad], dim=0) + return images + + +class Phi3VImageProcessor(BaseImageProcessor): + r""" + Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques + for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512) + + Args: + image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + num_crops: int = 1, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = True, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.num_crops = num_crops + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + self.do_convert_rgb = do_convert_rgb + + def calc_num_image_tokens(self, images: ImageInput): + """Calculate the number of image tokens for each image. + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + """ + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + images = [image.convert("RGB") for image in images] + # (H, W, C) + elems = [HD_transform(im, hd_num=self.num_crops) for im in images] + shapes = [[im.size[1], im.size[0]] for im in elems] + num_img_tokens = [int((h // 336 * w // 336 + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] + return num_img_tokens + + def calc_num_image_tokens_from_image_size(self, width, height): + """ + Calculate the number of image tokens for a given image size. + Args: + width (`int`): Width of the image. + height (`int`): Height of the image. + """ + new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops) + num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12) + return num_img_tokens + + def preprocess( + self, + images: ImageInput, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = None, + return_tensors: Optional[Union[str, TensorType]] = None, + ): + """ + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + """ + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + image_sizes = [] + img_processor = torchvision.transforms.Compose( + [torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(image_mean, image_std)] + ) + + # PIL images + # HD_transform pad images to size of multiiply of 336, 336 + # convert to RGB first + images = [image.convert("RGB") for image in images] + elems = [HD_transform(im, hd_num=self.num_crops) for im in images] + # tensor transform and normalize + hd_images = [img_processor(im) for im in elems] + # create global image + global_image = [ + torch.nn.functional.interpolate( + im.unsqueeze(0).float(), + size=(336, 336), + mode="bicubic", + ).to(im.dtype) + for im in hd_images + ] + + # [(3, h, w)], where h, w is multiple of 336 + shapes = [[im.size(1), im.size(2)] for im in hd_images] + num_img_tokens = [int(((h // 336) * (w // 336) + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] + # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336) + # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336) + hd_images_reshape = [ + im.reshape(1, 3, h // 336, 336, w // 336, 336) + .permute(0, 2, 4, 1, 3, 5) + .reshape(-1, 3, 336, 336) + .contiguous() + for im, (h, w) in zip(hd_images, shapes) + ] + # concat global image and local image + hd_images_reshape = [ + torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape) + ] + + # pad to max_num_crops + image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops + 1) for im in hd_images_reshape] + image_transformed = torch.stack(image_transformed, dim=0) + image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes] + padded_images = image_transformed + image_sizes = shapes + + data = {"pixel_values": padded_images, "image_sizes": image_sizes, "num_img_tokens": num_img_tokens} + + return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index c4049e431093..834b74712f31 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -20,280 +20,16 @@ import re from typing import List, Optional, Union -import numpy as np - -from ...image_processing_utils import BaseImageProcessor, BatchFeature +from ...image_processing_utils import BatchFeature from ...image_transforms import convert_to_rgb -from ...image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - ImageInput, - make_list_of_images, - valid_images, -) +from ...image_utils import ImageInput from ...processing_utils import ProcessorMixin from ...tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy -from ...utils import TensorType, logging, is_vision_available, is_torch_available, is_torchvision_available - - -if is_vision_available(): - from PIL import Image +from ...utils import TensorType, is_torch_available if is_torch_available(): import torch -if is_torchvision_available(): - import torchvision - - -logger = logging.get_logger(__name__) - - -def padding_336(b): - width, height = b.size - tar = int(np.ceil(height / 336) * 336) - top_padding = int((tar - height) / 2) - bottom_padding = tar - height - top_padding - left_padding = 0 - right_padding = 0 - b = torchvision.transforms.functional.pad( - b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255, 255, 255] - ) - - return b - - -def calc_padded_size(width, height, padding_unit=336): - target_height = int(np.ceil(height / padding_unit) * padding_unit) - top_padding = int((target_height - height) / 2) - bottom_padding = target_height - height - top_padding - left_padding = 0 - right_padding = 0 - padded_width = width + left_padding + right_padding - padded_height = height + top_padding + bottom_padding - return padded_width, padded_height - - -def HD_transform(img, hd_num=16): - width, height = img.size - trans = False - if width < height: - img = img.transpose(Image.TRANSPOSE) - trans = True - width, height = img.size - ratio = width / height - scale = 1 - while scale * np.ceil(scale / ratio) <= hd_num: - scale += 1 - scale -= 1 - new_w = int(scale * 336) - new_h = int(new_w / ratio) - - img = torchvision.transforms.functional.resize( - img, - [new_h, new_w], - ) - img = padding_336(img) - width, height = img.size - if trans: - img = img.transpose(Image.TRANSPOSE) - - return img - - -def calc_hd_transform_size(width, height, hd_num=16): - transposed = False - if width < height: - width, height = height, width - transposed = True - - ratio = width / height - scale = 1 - while scale * np.ceil(scale / ratio) <= hd_num: - scale += 1 - scale -= 1 - - new_width = int(scale * 336) - new_height = int(new_width / ratio) - - padded_width, padded_height = calc_padded_size(new_width, new_height) - - if transposed: - padded_width, padded_height = padded_height, padded_width - - return padded_width, padded_height - - -def pad_to_max_num_crops_tensor(images, max_crops=5): - """ - images: B x 3 x H x W, B<=max_crops - """ - B, _, H, W = images.shape - if B < max_crops: - pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device) - images = torch.cat([images, pad], dim=0) - return images - - -class Phi3VImageProcessor(BaseImageProcessor): - r""" - Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques - for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512) - - Args: - image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): - Mean to use if normalizing the image. This is a float or list of floats the length of the number of - channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): - Standard deviation to use if normalizing the image. This is a float or list of floats the length of the - number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. - Can be overridden by the `image_std` parameter in the `preprocess` method. - do_convert_rgb (`bool`, *optional*, defaults to `True`): - Whether to convert the image to RGB. - """ - - model_input_names = ["pixel_values"] - - def __init__( - self, - num_crops: int = 1, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = True, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.num_crops = num_crops - self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD - self.do_convert_rgb = do_convert_rgb - - def calc_num_image_tokens(self, images: ImageInput): - """Calculate the number of image tokens for each image. - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - """ - images = make_list_of_images(images) - - if not valid_images(images): - raise ValueError( - "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " - "torch.Tensor, tf.Tensor or jax.ndarray." - ) - - images = [image.convert("RGB") for image in images] - # (H, W, C) - elems = [HD_transform(im, hd_num=self.num_crops) for im in images] - shapes = [[im.size[1], im.size[0]] for im in elems] - num_img_tokens = [int((h // 336 * w // 336 + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] - return num_img_tokens - - def calc_num_image_tokens_from_image_size(self, width, height): - """ - Calculate the number of image tokens for a given image size. - Args: - width (`int`): Width of the image. - height (`int`): Height of the image. - """ - new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops) - num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12) - return num_img_tokens - - def preprocess( - self, - images: ImageInput, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = None, - return_tensors: Optional[Union[str, TensorType]] = None, - ): - """ - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to - `True`. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. - """ - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - - images = make_list_of_images(images) - - if not valid_images(images): - raise ValueError( - "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " - "torch.Tensor, tf.Tensor or jax.ndarray." - ) - - if do_convert_rgb: - images = [convert_to_rgb(image) for image in images] - - image_sizes = [] - img_processor = torchvision.transforms.Compose( - [torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(image_mean, image_std)] - ) - - # PIL images - # HD_transform pad images to size of multiiply of 336, 336 - # convert to RGB first - images = [image.convert("RGB") for image in images] - elems = [HD_transform(im, hd_num=self.num_crops) for im in images] - # tensor transform and normalize - hd_images = [img_processor(im) for im in elems] - # create global image - global_image = [ - torch.nn.functional.interpolate( - im.unsqueeze(0).float(), - size=(336, 336), - mode="bicubic", - ).to(im.dtype) - for im in hd_images - ] - - # [(3, h, w)], where h, w is multiple of 336 - shapes = [[im.size(1), im.size(2)] for im in hd_images] - num_img_tokens = [int(((h // 336) * (w // 336) + 1) * 144 + 1 + (h // 336 + 1) * 12) for h, w in shapes] - # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336) - # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336) - hd_images_reshape = [ - im.reshape(1, 3, h // 336, 336, w // 336, 336) - .permute(0, 2, 4, 1, 3, 5) - .reshape(-1, 3, 336, 336) - .contiguous() - for im, (h, w) in zip(hd_images, shapes) - ] - # concat global image and local image - hd_images_reshape = [ - torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape) - ] - - # pad to max_num_crops - image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops + 1) for im in hd_images_reshape] - image_transformed = torch.stack(image_transformed, dim=0) - image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes] - padded_images = image_transformed - image_sizes = shapes - - data = {"pixel_values": padded_images, "image_sizes": image_sizes, "num_img_tokens": num_img_tokens} - - return BatchFeature(data=data, tensor_type=return_tensors) - class Phi3VProcessor(ProcessorMixin): r""" From 97aa3087ca1bda3fad6d537556d95e0c89c15d5e Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 11 Feb 2025 15:44:16 +0000 Subject: [PATCH 08/48] Test commit to fix import structure --- src/transformers/models/phi3_5/__init__.py | 84 ++++++---------------- 1 file changed, 21 insertions(+), 63 deletions(-) diff --git a/src/transformers/models/phi3_5/__init__.py b/src/transformers/models/phi3_5/__init__.py index cdeca0d48e8d..3833c7e3237d 100644 --- a/src/transformers/models/phi3_5/__init__.py +++ b/src/transformers/models/phi3_5/__init__.py @@ -1,71 +1,29 @@ +# Copyright 2025 Microsoft and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import TYPE_CHECKING -from ...utils import ( - OptionalDependencyNotAvailable, - _LazyModule, - is_torch_available, - is_torchvision_available -) - - -_import_structure = { - "configuration_phi3_v": ["Phi3VConfig"], -} - -try: - if not is_torch_available(): - raise OptionalDependencyNotAvailable() -except OptionalDependencyNotAvailable: - pass -else: - _import_structure["modeling_phi3_v"] = [ - "Phi3VForCausalLM", - "Phi3VForSequenceClassification", - "Phi3VForTokenClassification", - "Phi3VModel", - "Phi3VPreTrainedModel", - ] - _import_structure["processing_phi3_v"] = ["Phi3VProcessor"] - try: - if not is_torchvision_available(): - raise OptionalDependencyNotAvailable() - except OptionalDependencyNotAvailable: - pass - else: - _import_structure["image_processing_phi3_v"] = ["Phi3VImageProcessor"] +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure if TYPE_CHECKING: - from .configuration_phi3_v import Phi3VConfig - - try: - if not is_torch_available(): - raise OptionalDependencyNotAvailable() - except OptionalDependencyNotAvailable: - pass - else: - from .modeling_phi3_v import ( - Phi3VForCausalLM, - Phi3VForSequenceClassification, - Phi3VForTokenClassification, - Phi3VModel, - Phi3VPreTrainedModel, - ) - from .processing_phi3_v import Phi3VProcessor - try: - if not is_torchvision_available(): - raise OptionalDependencyNotAvailable() - except OptionalDependencyNotAvailable: - pass - else: - from .image_processing_phi3_v import Phi3VImageProcessor - - + from .configuration_phi3_v import * + from .image_processing_phi3_v import * + from .modeling_phi3_v import * + from .processing_phi3_v import * else: import sys - sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) - -#from .configuration_phi3_v import * -#from .modeling_phi3_v import * -#from .processing_phi3_v import * + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) From 29e3545645dcc7b3718180d1ef618e6bd35adfa0 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 11 Feb 2025 15:54:14 +0000 Subject: [PATCH 09/48] More init cleanup --- src/transformers/models/phi3_5/configuration_phi3_v.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/models/phi3_5/configuration_phi3_v.py b/src/transformers/models/phi3_5/configuration_phi3_v.py index 92ac0ca725c1..bbf083d94b33 100644 --- a/src/transformers/models/phi3_5/configuration_phi3_v.py +++ b/src/transformers/models/phi3_5/configuration_phi3_v.py @@ -214,3 +214,6 @@ def _rope_scaling_validation(self): raise ValueError( f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" ) + + +__all__ = ["Phi3VConfig"] From 6b756c3912321ab7d6e4b282d042d67e14b3d092 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 11 Feb 2025 13:43:20 -0700 Subject: [PATCH 10/48] Now that I see the actual problem, here's the REAL final import errors fix --- .../models/phi3_5/image_processing_phi3_v.py | 3 +++ src/transformers/models/phi3_5/modeling_phi3_v.py | 9 +++++++++ src/transformers/models/phi3_5/processing_phi3_v.py | 3 +++ 3 files changed, 15 insertions(+) diff --git a/src/transformers/models/phi3_5/image_processing_phi3_v.py b/src/transformers/models/phi3_5/image_processing_phi3_v.py index dc06dbe23772..37e22f464568 100644 --- a/src/transformers/models/phi3_5/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_5/image_processing_phi3_v.py @@ -290,3 +290,6 @@ def preprocess( data = {"pixel_values": padded_images, "image_sizes": image_sizes, "num_img_tokens": num_img_tokens} return BatchFeature(data=data, tensor_type=return_tensors) + + +__all__ = ["Phi3VImageProcessor"] diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_5/modeling_phi3_v.py index 78dee4acdce5..d56892995b66 100644 --- a/src/transformers/models/phi3_5/modeling_phi3_v.py +++ b/src/transformers/models/phi3_5/modeling_phi3_v.py @@ -1941,3 +1941,12 @@ def forward( hidden_states=model_outputs.hidden_states, attentions=model_outputs.attentions, ) + + +__all__ = [ + "Phi3VForCausalLM", + "Phi3VForSequenceClassification", + "Phi3VForTokenClassification", + "Phi3VModel", + "Phi3VPreTrainedModel", +] diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index 834b74712f31..67d9155308d1 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -233,3 +233,6 @@ def model_input_names(self): tokenizer_input_names = self.tokenizer.model_input_names image_processor_input_names = self.image_processor.model_input_names return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) + + +__all__ = ["Phi3VProcessor"] From 01abfb257250e040d64d3b7c4ccdb4513ed49c35 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 11 Feb 2025 15:36:15 -0700 Subject: [PATCH 11/48] fixup changes --- src/transformers/models/phi3_5/image_processing_phi3_v.py | 3 ++- src/transformers/models/phi3_5/modeling_phi3_v.py | 3 ++- src/transformers/models/phi3_5/processing_phi3_v.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/phi3_5/image_processing_phi3_v.py b/src/transformers/models/phi3_5/image_processing_phi3_v.py index 37e22f464568..d37761a8daa3 100644 --- a/src/transformers/models/phi3_5/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_5/image_processing_phi3_v.py @@ -30,7 +30,7 @@ make_list_of_images, valid_images, ) -from ...utils import TensorType, logging, is_vision_available, is_torch_available, is_torchvision_available +from ...utils import TensorType, is_torch_available, is_torchvision_available, is_vision_available, logging if is_vision_available(): @@ -138,6 +138,7 @@ class Phi3VImageProcessor(BaseImageProcessor): for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512) Args: + num_crops (`int`, *optional*, defaults to 1): image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_5/modeling_phi3_v.py index d56892995b66..57b69185dd5a 100644 --- a/src/transformers/models/phi3_5/modeling_phi3_v.py +++ b/src/transformers/models/phi3_5/modeling_phi3_v.py @@ -24,7 +24,7 @@ import torch.nn.functional as F import torch.utils.checkpoint from torch import nn -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +from torch.nn import CrossEntropyLoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache @@ -35,6 +35,7 @@ SequenceClassifierOutputWithPast, TokenClassifierOutput, ) +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import PreTrainedModel from ...utils import ( add_code_sample_docstrings, diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_5/processing_phi3_v.py index 67d9155308d1..23dffdfb2b43 100644 --- a/src/transformers/models/phi3_5/processing_phi3_v.py +++ b/src/transformers/models/phi3_5/processing_phi3_v.py @@ -21,11 +21,11 @@ from typing import List, Optional, Union from ...image_processing_utils import BatchFeature -from ...image_transforms import convert_to_rgb from ...image_utils import ImageInput from ...processing_utils import ProcessorMixin from ...tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy -from ...utils import TensorType, is_torch_available +from ...utils import TensorType, is_torch_available + if is_torch_available(): import torch From 6d1f8d67fa18999997abd78c57d36e5fab0ada0e Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 11 Feb 2025 16:59:19 -0700 Subject: [PATCH 12/48] renamed from phi3_5 to phi3_v --- docs/source/en/_toctree.yml | 2 +- src/transformers/models/__init__.py | 2 +- src/transformers/models/auto/configuration_auto.py | 4 ++-- src/transformers/models/auto/image_processing_auto.py | 2 +- src/transformers/models/auto/modeling_auto.py | 8 ++++---- src/transformers/models/{phi3_5 => phi3_v}/__init__.py | 0 .../models/{phi3_5 => phi3_v}/configuration_phi3_v.py | 0 .../models/{phi3_5 => phi3_v}/image_processing_phi3_v.py | 0 .../models/{phi3_5 => phi3_v}/modeling_phi3_v.py | 2 +- .../models/{phi3_5 => phi3_v}/processing_phi3_v.py | 0 10 files changed, 10 insertions(+), 10 deletions(-) rename src/transformers/models/{phi3_5 => phi3_v}/__init__.py (100%) rename src/transformers/models/{phi3_5 => phi3_v}/configuration_phi3_v.py (100%) rename src/transformers/models/{phi3_5 => phi3_v}/image_processing_phi3_v.py (100%) rename src/transformers/models/{phi3_5 => phi3_v}/modeling_phi3_v.py (99%) rename src/transformers/models/{phi3_5 => phi3_v}/processing_phi3_v.py (100%) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 1d1a4964da63..ce704c7df21b 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -1091,7 +1091,7 @@ title: Perceiver - local: model_doc/perception_lm title: PerceptionLM - - local: model_doc/phi3_5 + - local: model_doc/phi3_v title: Phi-3.5-vision-instruct - local: model_doc/phi4_multimodal title: Phi4 Multimodal diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index d80703c17c9b..0d36714475b8 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -253,7 +253,7 @@ from .persimmon import * from .phi import * from .phi3 import * - from .phi3_5 import * + from .phi3_v import * from .phi4_multimodal import * from .phimoe import * from .phobert import * diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 99dd22ac2482..dcc2671b74c5 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -295,7 +295,7 @@ ("persimmon", "PersimmonConfig"), ("phi", "PhiConfig"), ("phi3", "Phi3Config"), - ("phi3_5", "Phi3VConfig"), + ("phi3_v", "Phi3VConfig"), ("phi4_multimodal", "Phi4MultimodalConfig"), ("phimoe", "PhimoeConfig"), ("pix2struct", "Pix2StructConfig"), @@ -725,7 +725,7 @@ ("persimmon", "Persimmon"), ("phi", "Phi"), ("phi3", "Phi3"), - ("phi3_5", "Phi3V"), + ("phi3_v", "Phi3V"), ("phi4_multimodal", "Phi4Multimodal"), ("phimoe", "Phimoe"), ("phobert", "PhoBERT"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index c10e8d7b75a3..d4f8c2bc7204 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -146,7 +146,7 @@ ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")), ("perceiver", ("PerceiverImageProcessor", "PerceiverImageProcessorFast")), ("perception_lm", (None, "PerceptionLMImageProcessorFast")), - ("phi3_5", ("Phi3VImageProcessor",)), + ("phi3_v", ("Phi3VImageProcessor",)), ("phi4_multimodal", (None, "Phi4MultimodalImageProcessorFast")), ("pix2struct", ("Pix2StructImageProcessor", None)), ("pixtral", ("PixtralImageProcessor", "PixtralImageProcessorFast")), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index c2fd4c3f8ceb..14b2747f035c 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -294,7 +294,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonModel"), ("phi", "PhiModel"), ("phi3", "Phi3Model"), - ("phi3_5", "Phi3VModel"), + ("phi3_v", "Phi3VModel"), ("phi4_multimodal", "Phi4MultimodalModel"), ("phimoe", "PhimoeModel"), ("pixtral", "PixtralVisionModel"), @@ -695,7 +695,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForCausalLM"), ("phi", "PhiForCausalLM"), ("phi3", "Phi3ForCausalLM"), - ("phi3_5", "Phi3VForCausalLM"), + ("phi3_v", "Phi3VForCausalLM"), ("phi4_multimodal", "Phi4MultimodalForCausalLM"), ("phimoe", "PhimoeForCausalLM"), ("plbart", "PLBartForCausalLM"), @@ -1241,7 +1241,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForSequenceClassification"), ("phi", "PhiForSequenceClassification"), ("phi3", "Phi3ForSequenceClassification"), - ("phi3_5", "Phi3VForSequenceClassification"), + ("phi3_v", "Phi3VForSequenceClassification"), ("phimoe", "PhimoeForSequenceClassification"), ("plbart", "PLBartForSequenceClassification"), ("qdqbert", "QDQBertForSequenceClassification"), @@ -1443,7 +1443,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("persimmon", "PersimmonForTokenClassification"), ("phi", "PhiForTokenClassification"), ("phi3", "Phi3ForTokenClassification"), - ("phi3_5", "Phi3VForTokenClassification"), + ("phi3_v", "Phi3VForTokenClassification"), ("qdqbert", "QDQBertForTokenClassification"), ("qwen2", "Qwen2ForTokenClassification"), ("qwen2_moe", "Qwen2MoeForTokenClassification"), diff --git a/src/transformers/models/phi3_5/__init__.py b/src/transformers/models/phi3_v/__init__.py similarity index 100% rename from src/transformers/models/phi3_5/__init__.py rename to src/transformers/models/phi3_v/__init__.py diff --git a/src/transformers/models/phi3_5/configuration_phi3_v.py b/src/transformers/models/phi3_v/configuration_phi3_v.py similarity index 100% rename from src/transformers/models/phi3_5/configuration_phi3_v.py rename to src/transformers/models/phi3_v/configuration_phi3_v.py diff --git a/src/transformers/models/phi3_5/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py similarity index 100% rename from src/transformers/models/phi3_5/image_processing_phi3_v.py rename to src/transformers/models/phi3_v/image_processing_phi3_v.py diff --git a/src/transformers/models/phi3_5/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py similarity index 99% rename from src/transformers/models/phi3_5/modeling_phi3_v.py rename to src/transformers/models/phi3_v/modeling_phi3_v.py index 57b69185dd5a..531e7c9b8d9d 100644 --- a/src/transformers/models/phi3_5/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -393,7 +393,7 @@ def _get_unpad_data(attention_mask): ) -# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3_5, Gemma->Phi3V +# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3_v, Gemma->Phi3V class Phi3RotaryEmbedding(nn.Module): def __init__(self, config: Phi3VConfig, device=None): super().__init__() diff --git a/src/transformers/models/phi3_5/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py similarity index 100% rename from src/transformers/models/phi3_5/processing_phi3_v.py rename to src/transformers/models/phi3_v/processing_phi3_v.py From afa0bcc70028ce75291e16813762f8fa7d19a55d Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 23 Feb 2025 16:16:16 -0700 Subject: [PATCH 13/48] Reverted Phi3RotaryEmbedding due to change in copied code --- .../models/phi3_v/modeling_phi3_v.py | 59 +++++-------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 531e7c9b8d9d..9db5487975a1 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -393,53 +393,27 @@ def _get_unpad_data(attention_mask): ) -# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3_v, Gemma->Phi3V +# TODO Used to be copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding, but the signature changed class Phi3RotaryEmbedding(nn.Module): - def __init__(self, config: Phi3VConfig, device=None): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): super().__init__() - # BC: "rope_type" was originally "type" - if hasattr(config, "rope_scaling") and config.rope_scaling is not None: - self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) - else: - self.rope_type = "default" - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - - inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device) - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.original_inv_freq = self.inv_freq - def _dynamic_frequency_update(self, position_ids, device): - """ - dynamic RoPE layers should recompute `inv_freq` in the following situations: - 1 - growing beyond the cached sequence length (allow scaling) - 2 - the current sequence length is in the original scale (avoid losing precision with small sequences) - """ - seq_len = torch.max(position_ids) + 1 - if seq_len > self.max_seq_len_cached: # growth - inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len) - self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation - self.max_seq_len_cached = seq_len - - if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset - # This .to() is needed if the model has been moved to a device after being initialized (because - # the buffer is automatically moved, but not the original copy) - self.original_inv_freq = self.original_inv_freq.to(device) - self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) - self.max_seq_len_cached = self.original_max_seq_len + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.register_buffer("inv_freq", None, persistent=False) @torch.no_grad() - def forward(self, x, position_ids): - if "dynamic" in self.rope_type: - self._dynamic_frequency_update(position_ids, device=x.device) - - # Core RoPE block + def forward(self, x, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if self.inv_freq is None: + self.inv_freq = 1.0 / ( + self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim) + ) inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) position_ids_expanded = position_ids[:, None, :].float() - # Force float32 (see https://github.com/huggingface/transformers/pull/29285) + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 device_type = x.device.type device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" with torch.autocast(device_type=device_type, enabled=False): @@ -447,11 +421,6 @@ def forward(self, x, position_ids): emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() sin = emb.sin() - - # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention - cos = cos * self.attention_scaling - sin = sin * self.attention_scaling - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) From ca35ff4d9c76cb10b65b8f7800d0b139b5720301 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 24 Feb 2025 16:39:59 -0700 Subject: [PATCH 14/48] Erroneous call to DynamicCache class amended --- src/transformers/models/phi3_v/modeling_phi3_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 9db5487975a1..76c9ed1c979e 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -1657,7 +1657,7 @@ def prepare_inputs_for_generation( if isinstance(past_key_values, Cache): cache_length = past_key_values.get_seq_length() past_length = past_key_values.seen_tokens - max_cache_length = past_key_values.get_max_length() + max_cache_length = past_key_values.get_max_cache_shape() else: cache_length = past_length = past_key_values[0][0].shape[2] max_cache_length = None From 10d3c30602a44ee1402d351118d08811bb8a66e8 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 24 Feb 2025 17:51:04 -0700 Subject: [PATCH 15/48] make fixup changes --- .../models/phi3_v/modeling_phi3_v.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 76c9ed1c979e..b82458edceaf 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -35,7 +35,6 @@ SequenceClassifierOutputWithPast, TokenClassifierOutput, ) -from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import PreTrainedModel from ...utils import ( add_code_sample_docstrings, @@ -1794,17 +1793,20 @@ def forward( if self.config.pad_token_id is None and batch_size != 1: raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") if self.config.pad_token_id is None: - sequence_lengths = -1 + last_non_pad_token = -1 + elif input_ids is not None: + # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id + non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) + token_indices = torch.arange(input_ids.shape[-1], device=logits.device) + last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) else: - if input_ids is not None: - # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility - sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 - sequence_lengths = sequence_lengths % input_ids.shape[-1] - sequence_lengths = sequence_lengths.to(logits.device) - else: - sequence_lengths = -1 + last_non_pad_token = -1 + logger.warning_once( + f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be " + "unexpected if using padding tokens in conjunction with `inputs_embeds.`" + ) - pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token] loss = None if labels is not None: From 95ca9c77786d77a2ca46d9e486c15d08bdadf146 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Fri, 28 Feb 2025 04:24:17 -0700 Subject: [PATCH 16/48] Alright, @Rocketknight1 I copied over qwen2's test modeling file to try and get things working in general, but there's a handful of tests I just can't get to pass! They all seem to be related to the GenerationTesterMixin class, and in cross-referencing pixtral's(a comparable model) test_modeling code, that class isn't included there. Ultimately I'm stumped as to why it's failing or if it's even needed here --- .../models/phi3_v/modeling_phi3_v.py | 4 +- tests/models/phi3_v/__init__.py | 0 tests/models/phi3_v/test_modeling_phi3_v.py | 1038 +++++++++++++++++ 3 files changed, 1041 insertions(+), 1 deletion(-) create mode 100644 tests/models/phi3_v/__init__.py create mode 100644 tests/models/phi3_v/test_modeling_phi3_v.py diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index b82458edceaf..1b6906bbe2c9 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -28,6 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -1498,7 +1499,7 @@ def forward( ) -class Phi3VForCausalLM(Phi3VPreTrainedModel): +class Phi3VForCausalLM(Phi3VPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3V @@ -1552,6 +1553,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, + **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" Args: diff --git a/tests/models/phi3_v/__init__.py b/tests/models/phi3_v/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py new file mode 100644 index 000000000000..e644d31907b8 --- /dev/null +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -0,0 +1,1038 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Testing suite for the PyTorch Phi3V model.""" + +import gc +import unittest + +import pytest +from packaging import version + +from transformers import AutoTokenizer, Phi3VConfig, is_torch_available, set_seed +from transformers.generation.configuration_utils import GenerationConfig +from transformers.testing_utils import ( + backend_empty_cache, + require_bitsandbytes, + require_flash_attn, + require_torch, + require_torch_gpu, + require_torch_sdpa, + slow, + torch_device, +) + +from ...generation.test_utils import GenerationTesterMixin +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, ids_tensor +from ...test_pipeline_mixin import PipelineTesterMixin + + +if is_torch_available(): + import torch + + from transformers import ( + Phi3VForCausalLM, + Phi3VForSequenceClassification, + Phi3VForTokenClassification, + Phi3VModel, + ) + + +class Phi3VModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + max_window_layers=3, + use_sliding_window=True, + sliding_window=50, + num_attention_heads=4, + num_key_value_heads=2, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + pad_token_id=0, + bos_token_id=1, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.max_window_layers = max_window_layers + self.use_sliding_window = use_sliding_window + self.sliding_window = sliding_window + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.scope = scope + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.prepare_config_and_inputs + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = torch.tril(torch.ones_like(input_ids).to(torch_device)) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = self.get_config() + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def get_config(self): + return Phi3VConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + max_window_layers=self.max_window_layers, + use_sliding_window=self.use_sliding_window, + sliding_window=self.sliding_window, + num_attention_heads=self.num_attention_heads, + num_key_value_heads=self.num_key_value_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + is_decoder=False, + initializer_range=self.initializer_range, + pad_token_id=self.pad_token_id, + bos_token_id=self.bos_token_id, + ) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_model with Llama->Phi3V + def create_and_check_model( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + model = Phi3VModel(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_model_as_decoder with Llama->Phi3V + def create_and_check_model_as_decoder( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.add_cross_attention = True + model = Phi3VModel(config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + ) + result = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + ) + result = model(input_ids, attention_mask=input_mask) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_for_causal_lm with Llama->Phi3V + def create_and_check_for_causal_lm( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + model = Phi3VForCausalLM(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_decoder_model_past_large_inputs with Llama->Phi3V + def create_and_check_decoder_model_past_large_inputs( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.is_decoder = True + config.add_cross_attention = True + model = Phi3VForCausalLM(config=config) + model.to(torch_device) + model.eval() + + # first forward pass + outputs = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=True, + ) + past_key_values = outputs.past_key_values + + # create hypothetical multiple next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) + next_mask = ids_tensor((self.batch_size, 3), vocab_size=2) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + next_attention_mask = torch.cat([input_mask, next_mask], dim=-1) + + output_from_no_past = model( + next_input_ids, + attention_mask=next_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_hidden_states=True, + )["hidden_states"][0] + output_from_past = model( + next_tokens, + attention_mask=next_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + output_hidden_states=True, + )["hidden_states"][0] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() + + self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) + + # test that outputs are equal for slice + self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.prepare_config_and_inputs_for_common + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + ) = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +# Copied from tests.models.mistral.test_modeling_mistral.MistralModelTest with Mistral->Phi3V +class Phi3VModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): + all_model_classes = ( + ( + Phi3VModel, + Phi3VForCausalLM, + Phi3VForSequenceClassification, + Phi3VForTokenClassification, + ) + if is_torch_available() + else () + ) + pipeline_model_mapping = ( + { + "feature-extraction": Phi3VModel, + "text-classification": Phi3VForSequenceClassification, + "token-classification": Phi3VForTokenClassification, + "text-generation": Phi3VForCausalLM, + "zero-shot": Phi3VForSequenceClassification, + } + if is_torch_available() + else {} + ) + test_headmasking = False + test_pruning = False + fx_compatible = False # Broken by attention refactor cc @Cyrilvallez + + def is_pipeline_test_to_skip( + self, + pipeline_test_case_name, + config_class, + model_architecture, + tokenizer_name, + image_processor_name, + feature_extractor_name, + processor_name, + ): + return True + + def setUp(self): + self.model_tester = Phi3VModelTester(self) + self.config_tester = ConfigTester(self, config_class=Phi3VConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_model_various_embeddings(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + for type in ["absolute", "relative_key", "relative_key_query"]: + config_and_inputs[0].position_embedding_type = type + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_torch_fx_output_loss(self): + super().test_torch_fx_output_loss() + + def test_Phi3V_sequence_classification_model(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + print(config) + config.num_labels = 3 + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.type_sequence_label_size) + model = Phi3VForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + def test_Phi3V_sequence_classification_model_for_single_label(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + config.problem_type = "single_label_classification" + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.type_sequence_label_size) + model = Phi3VForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + def test_Phi3V_sequence_classification_model_for_multi_label(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + config.problem_type = "multi_label_classification" + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor( + [self.model_tester.batch_size, config.num_labels], self.model_tester.type_sequence_label_size + ).to(torch.float) + model = Phi3VForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + # Copied from tests.models.llama.test_modeling_llama.LlamaModelTest.test_llama_token_classification_model with Llama->Phi3V,llama->Phi3V + def test_Phi3V_token_classification_model(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + token_labels = ids_tensor([self.model_tester.batch_size, self.model_tester.seq_length], config.num_labels) + model = Phi3VForTokenClassification(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=token_labels) + self.assertEqual( + result.logits.shape, + (self.model_tester.batch_size, self.model_tester.seq_length, self.model_tester.num_labels), + ) + + @unittest.skip(reason="Phi3V buffers include complex numbers, which breaks this test") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="Phi3V uses GQA on all models so the KV cache is a non standard format") + def test_past_key_values_format(self): + pass + + @require_flash_attn + @require_torch_gpu + @pytest.mark.flash_attn_test + @slow + def test_flash_attn_2_inference_equivalence_right_padding(self): + self.skipTest(reason="Phi3V flash attention does not support right padding") + + +@require_torch +class Phi3VIntegrationTest(unittest.TestCase): + @slow + def test_model_450m_logits(self): + input_ids = [1, 306, 4658, 278, 6593, 310, 2834, 338] + model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto") + input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + with torch.no_grad(): + out = model(input_ids).logits.float().cpu() + # Expected mean on dim = -1 + EXPECTED_MEAN = torch.tensor([[-1.9537, -1.6193, -1.4123, -1.4673, -1.8511, -1.9309, -1.9826, -2.1776]]) + torch.testing.assert_close(out.mean(-1), EXPECTED_MEAN, rtol=1e-2, atol=1e-2) + # slicing logits[0, 0, 0:30] + EXPECTED_SLICE = torch.tensor([3.2025, 7.1265, 4.6058, 3.6423, 1.6357, 3.9265, 5.1883, 5.8760, 2.7942, 4.4823, 3.2571, 2.1063, 3.4275, 4.2028, 1.9767, 5.2115, 6.6756, 6.3999, 6.0483, 5.7378, 5.6660, 5.2298, 5.4103, 5.1248, 5.4376, 2.4570, 2.6107, 5.4039, 2.8077, 4.7777]) # fmt: skip + print(out[0, 0, :30]) + torch.testing.assert_close(out[0, 0, :30], EXPECTED_SLICE, rtol=1e-4, atol=1e-4) + + del model + backend_empty_cache(torch_device) + gc.collect() + + @slow + def test_model_450m_generation(self): + EXPECTED_TEXT_COMPLETION = ( + """My favourite condiment is 100% natural, organic and vegan. I love to use it in my cooking and I""" + ) + prompt = "My favourite condiment is " + tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-0.5B", use_fast=False) + model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto") + input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) + + # greedy generation outputs + generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) + text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + self.assertEqual(EXPECTED_TEXT_COMPLETION, text) + + del model + backend_empty_cache(torch_device) + gc.collect() + + @require_bitsandbytes + @slow + @require_flash_attn + @pytest.mark.flash_attn_test + def test_model_450m_long_prompt(self): + EXPECTED_OUTPUT_TOKEN_IDS = [306, 338] + # An input with 4097 tokens that is above the size of the sliding window + input_ids = [1] + [306, 338] * 2048 + model = Phi3VForCausalLM.from_pretrained( + "Qwen/Phi3V-0.5B", + device_map="auto", + load_in_4bit=True, + attn_implementation="flash_attention_2", + ) + input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + # Assisted generation + assistant_model = model + assistant_model.generation_config.num_assistant_tokens = 2 + assistant_model.generation_config.num_assistant_tokens_schedule = "constant" + generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + del assistant_model + del model + backend_empty_cache(torch_device) + gc.collect() + + @slow + @require_torch_sdpa + def test_model_450m_long_prompt_sdpa(self): + EXPECTED_OUTPUT_TOKEN_IDS = [306, 338] + # An input with 4097 tokens that is above the size of the sliding window + input_ids = [1] + [306, 338] * 2048 + model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto", attn_implementation="sdpa") + input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) + generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + # Assisted generation + assistant_model = model + assistant_model.generation_config.num_assistant_tokens = 2 + assistant_model.generation_config.num_assistant_tokens_schedule = "constant" + generated_ids = assistant_model.generate(input_ids, max_new_tokens=4, temperature=0) + self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) + + del assistant_model + + backend_empty_cache(torch_device) + gc.collect() + + EXPECTED_TEXT_COMPLETION = ( + "My favourite condiment is 100% natural, organic and vegan. I love to use it in my cooking and I" + ) + prompt = "My favourite condiment is " + tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-0.5B", use_fast=False) + + input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) + + # greedy generation outputs + generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) + text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + self.assertEqual(EXPECTED_TEXT_COMPLETION, text) + + @slow + def test_speculative_generation(self): + EXPECTED_TEXT_COMPLETION = ( + "My favourite condiment is 100% natural honey, and I always like to use it in my recipes. I love" + ) + prompt = "My favourite condiment is " + tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-7B", use_fast=False) + model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto", torch_dtype=torch.float16) + assistant_model = Phi3VForCausalLM.from_pretrained( + "Qwen/Phi3V-0.5B", device_map="auto", torch_dtype=torch.float16 + ) + input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) + + # greedy generation outputs + set_seed(0) + generated_ids = model.generate( + input_ids, max_new_tokens=20, do_sample=True, temperature=0.3, assistant_model=assistant_model + ) + text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) + self.assertEqual(EXPECTED_TEXT_COMPLETION, text) + + del model + backend_empty_cache(torch_device) + gc.collect() + + @slow + def test_export_static_cache(self): + if version.parse(torch.__version__) < version.parse("2.4.0"): + self.skipTest(reason="This test requires torch >= 2.4 to run.") + + from transformers.integrations.executorch import ( + TorchExportableModuleWithStaticCache, + convert_and_export_with_cache, + ) + + qwen_model = "Qwen/Phi3V-0.5B" + + tokenizer = AutoTokenizer.from_pretrained(qwen_model, pad_token="", padding_side="right") + EXPECTED_TEXT_COMPLETION = [ + "My favourite condiment is 100% natural, organic, gluten free, vegan, and free from preservatives. I" + ] + max_generation_length = tokenizer(EXPECTED_TEXT_COMPLETION, return_tensors="pt", padding=True)[ + "input_ids" + ].shape[-1] + + # Load model + device = "cpu" + dtype = torch.bfloat16 + cache_implementation = "static" + attn_implementation = "sdpa" + batch_size = 1 + model = Phi3VForCausalLM.from_pretrained( + qwen_model, + device_map=device, + torch_dtype=dtype, + attn_implementation=attn_implementation, + generation_config=GenerationConfig( + use_cache=True, + cache_implementation=cache_implementation, + max_length=max_generation_length, + cache_config={ + "batch_size": batch_size, + "max_cache_len": max_generation_length, + }, + ), + ) + + prompt = ["My favourite condiment is "] + prompt_tokens = tokenizer(prompt, return_tensors="pt", padding=True).to(model.device) + prompt_token_ids = prompt_tokens["input_ids"] + max_new_tokens = max_generation_length - prompt_token_ids.shape[-1] + + # Static Cache + export + exported_program = convert_and_export_with_cache(model) + ep_generated_ids = TorchExportableModuleWithStaticCache.generate( + exported_program=exported_program, prompt_token_ids=prompt_token_ids, max_new_tokens=max_new_tokens + ) + ep_generated_text = tokenizer.batch_decode(ep_generated_ids, skip_special_tokens=True) + self.assertEqual(EXPECTED_TEXT_COMPLETION, ep_generated_text) + + @require_flash_attn + @slow + def test_3b_generation(self): + model_id = "Qwen/Phi3V.5-3B" + tokenizer = AutoTokenizer.from_pretrained(model_id) + model = Phi3VForCausalLM.from_pretrained( + model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048 + ).to(torch_device) + # we need a long text to test sliding window + # fmt: off + LONG_TEXT = """The Warring States period in Chinese history (c. 475 – 221 BC) comprises the final centuries of the Zhou dynasty (c. 1046 – 256 BC), which were characterized by warfare, bureaucratic and military reform, and political consolidation. It followed the Spring and Autumn period and concluded with the wars of conquest that saw the state of Qin annex each of the other contender states by 221 BC and found the Qin dynasty, the first imperial dynastic state in East Asian history. + + +While scholars have identified several different dates as marking the beginning of the Warring States period, Sima Qian's choice of 475 BC is the most often cited. The era largely corresponds to the second half of the Eastern Zhou period, where the king of Zhou formally ruled as Chinese sovereign, but had lost political power and functioned in practice as a figurehead. This dynamic served as the backdrop for the machinations of the eponymous Warring States. The label "Warring States period" derives from the Record of the Warring States, a work of history compiled during the early Han dynasty (202 BC – 220 AD). + + +Geography + +The political geography of the era was dominated by the Seven Warring States, namely: + + +Besides these seven major states other smaller states survived into the period. They include: + + +Periodisation + +The eastward flight of the Zhou court in 771 BC marks the start of the Spring and Autumn period. No one single incident or starting point inaugurated the Warring States era. The political situation of the period represented a culmination of historical trends of conquest and annexation which also characterised the Spring and Autumn period. As a result, there is some controversy as to the beginning of the era. Proposed starting points include: + + +History + +Background and formation + +The Eastern Zhou dynasty began its fall around 5th century BC. As their influence waned, they had to rely on armies in allied states rather than their own military force. Hundreds of smaller polities coalesced into seven major states which included: Chu, Han, Qin, Wei, Yan, Qi and Zhao. However, there eventually was a shift in alliances because each state's ruler wanted independence. This caused hundreds of wars between 535 and 286 BC. The victorious state would have overall rule and control in China. + + +The system of feudal states created by the Western Zhou dynasty underwent enormous changes after 771 BC with the flight of the Zhou court to modern-day Luoyang and the diminution of its relevance and power. The Spring and Autumn period led to a few states gaining power at the expense of many others, the latter no longer able to depend on central authority for legitimacy or protection. During the Warring States period, many rulers claimed the Mandate of Heaven to justify their conquest of other states and spread their influence. + + +The struggle for hegemony eventually created a state system dominated by several large states, such as Jin, Chu, Qin, Yan, and Qi, while the smaller states of the Central Plain tended to be their satellites and tributaries. Other major states also existed, such as Wu and Yue in the southeast. The last decades of the Spring and Autumn era were marked by increased stability, as the result of peace negotiations between Jin and Chu which established their respective spheres of influence. This situation ended with the partition of Jin, whereby the state was divided between the houses of Han, Zhao and Wei, leading to the seven major warring states. + + +Partition of Jin (453–403 BC) + +The rulers of Jin had steadily lost political powers since the middle of the 6th century BC to their nominally subordinate nobles and military commanders, a situation arising from the traditions of the Jin which forbade the enfeoffment of relatives of the ducal house. This allowed other clans to gain fiefs and military authority, and decades of internecine struggle led to the establishment of four major families, the Han, Zhao, Wei and Zhi. + + +The Battle of Jinyang saw the allied Han, Zhao and Wei destroy the Zhi family (453 BC) and their lands were distributed among them. With this, they became the de facto rulers of most of Jin's territory, though this situation would not be officially recognised until half a century later. The Jin division created a political vacuum that enabled during the first 50 years expansion of Chu and Yue northward and Qi southward. Qin increased its control of the local tribes and began its expansion southwest to Sichuan. + + +Early Warring States + +The three Jins recognized (403–364 BC) + +In 403 BC, the court of King Weilie of Zhou officially recognized Zhao, Wei and Han as immediate vassals, thereby raising them to the same rank as the other warring states. + + +From before 405 until 383 BC the three Jins were united under the leadership of Wei and expanded in all directions. The most important figure was Marquess Wen of Wei (445–396 BC). In 408–406 BC he conquered the State of Zhongshan to the northeast on the other side of Zhao. At the same time he pushed west across the Yellow River to the Luo River taking the area of Xihe (literally 'west of the river'). + + +The growing power of Wei caused Zhao to back away from the alliance. In 383 BC it moved its capital to Handan and attacked the small state of Wey. Wey appealed to Wei which attacked Zhao on the western side. Being in danger, Zhao called in Chu. As usual, Chu used this as a pretext to annex territory to its north, but the diversion allowed Zhao to occupy a part of Wei. This conflict marked the end of the power of the united Jins and the beginning a period of shifting alliances and wars on several fronts. + + +In 376 BC, the states of Han, Wei and Zhao deposed Duke Jing of Jin and divided the last remaining Jin territory between themselves, which marked the final end of the Jin state. + + +In 370 BC, Marquess Wu of Wei died without naming a successor, which led to a war of succession. After three years of civil war, Zhao from the north and Han from the south invaded Wei. On the verge of conquering Wei, the leaders of Zhao and Han fell into disagreement about what to do with Wei, and both armies abruptly retreated. As a result, King Hui of Wei (still a Marquess at the time) was able to ascend the throne of Wei. + + +Zhao extended from the Shanxi plateau across the plain to the borders of Qi. Wei reached east to Qi, Lu, and Song. To the south, the weaker state of Han held the east–west part of the Yellow River valley, surrounded the Zhou royal domain at Luoyang and held an area north of Luoyang called Shangdang. + + +Qi resurgence under Tian (379–340 BC) + +Duke Kang of Qi died in 379 BC with no heir from the house of Jiang, which had ruled Qi since the state's founding. The throne instead passed to the future King Wei, from the house of Tian. The Tian had been very influential at court towards the end of Jiang rule, and now openly assumed power. + + +The new ruler set about reclaiming territories that had been lost to other states. He launched a successful campaign against Zhao, Wey and Wei, once again extending Qi territory to the Great Wall. Sima Qian writes that the other states were so awestruck that nobody dared attack Qi for more than 20 years. The demonstrated military prowess also had a calming effect on Qi's own population, which experienced great domestic tranquility during Wei's reign. + + +By the end of King Wei's reign, Qi had become the strongest of the states and proclaimed itself "king"; establishing independence from the Zhou dynasty (see below). + + +Wars of Wei + +King Hui of Wei (370–319 BC) set about restoring the state. In 362–359 BC he exchanged territories with Han and Zhao in order to make the boundaries of the three states more rational. + + +In 364 BC, Wei was defeated by Qin at the Battle of Shimen and was only saved by the intervention of Zhao. Qin won another victory in 362 BC. In 361 BC the Wei capital was moved east to Daliang to be out of the reach of Qin. + + +In 354 BC, King Hui of Wei started a large-scale attack on Zhao. By 353 BC, Zhao was losing badly and its capital, Handan, was under siege. The state of Qi intervened. The famous Qi strategist, Sun Bin the great-great-great-grandson of Sun Tzu, the author of the Art of War, proposed to attack the Wei capital while the Wei army was tied up besieging Zhao. The strategy was a success; the Wei army hastily moved south to protect its capital, was caught on the road and decisively defeated at the Battle of Guiling. The battle is remembered in the second of the Thirty-Six Stratagems, "besiege Wei, save Zhao"—meaning to attack a vulnerable spot to relieve pressure at another point. + + +Domestically, King Hui patronized philosophy and the arts, and is perhaps best remembered for hosting the Confucian philosopher Mencius at his court; their conversations form the first two chapters of the book which bears Meng Zi's name. + + +Dukes become kings + +Qi and Wei became kingdoms (344 BC) + +The title of king (wang, 王) was held by figurehead rulers of the Zhou dynasty, while the rulers of most states held the title of duke (gong, 公) or marquess (hou, 侯). A major exception was Chu, whose rulers were called kings since King Wu of Chu started using the title c. 703 BC. + + +In 344 BC the rulers of Qi and Wei mutually recognized each other as kings: King Wei of Qi and King Hui of Wei, in effect declaring their independence from the Zhou court. This marked a major turning point: unlike those in the Spring and Autumn period, the new generation of rulers ascending the thrones in the Warring States period would not entertain even the pretence of being vassals of the Zhou dynasty, instead proclaiming themselves fully independent kingdoms. + + +Shang Yang reforms Qin (356–338 BC) + +During the early Warring States period Qin generally avoided conflicts with the other states. This changed during the reign of Duke Xiao, when prime minister Shang Yang made centralizing and authoritarian reforms in accordance with his Legalist philosophy between the years 356 and 338 BC. + + +Shang introduced land reforms, privatized land, rewarded farmers who exceeded harvest quotas, enslaved farmers who failed to meet quotas, and used enslaved subjects as rewards for those who met government policies. As manpower was short in Qin relative to the other states at the time, Shang enacted policies to increase its manpower. As Qin peasants were recruited into the military, he encouraged active immigration of peasants from other states into Qin as a replacement workforce; this policy simultaneously increased the manpower of Qin and weakened the manpower of Qin's rivals. + + +Shang made laws forcing citizens to marry at a young age and passed tax laws to encourage raising multiple children. He also enacted policies to free convicts who worked in opening wastelands for agriculture. Shang abolished primogeniture and created a double tax on households that had more than one son living in the household, to break up large clans into nuclear families. Shang also moved the capital to reduce the influence of nobles on the administration. + + +The rise of Qin was recognized by the royal court, and in 343 BC the king conferred the title of Count (伯 Bó) on Duke Xiao. As was customary, a conference was hosted which the feudal lords attended, and during which the Son of Heaven bestowed the title. + + +After the reforms Qin became much more aggressive. In 340 Qin took land from Wèi after it had been defeated by Qi. In 316 Qin conquered Shu and Ba in Sichuan to the southwest. Development of this area took a long time but slowly added greatly to Qin's wealth and power. + + +Qin defeats Wei (341–340 BC) + +In 341 BC, Wei attacked Han. Qi allowed Han to be nearly defeated and then intervened. The generals from the Battle of Guiling met again (Sun Bin and Tian Ji versus Pang Juan), using the same tactic, attacking Wei's capital. Sun Bin feigned a retreat and then turned on the overconfident Wei troops and decisively defeated them at the Battle of Maling. After the battle all three of the Jin successor states appeared before King Xuan of Qi, pledging their loyalty. + + +In the following year Qin attacked the weakened Wei. Wei was devastatingly defeated and ceded a large part of its territory in return for truce. With Wei severely weakened, Qi and Qin became the dominant states in China. + + +Wei came to rely on Qi for protection, with King Hui of Wei meeting King Xuan of Qi on two occasions. After Hui's death, his successor King Xiang also established a good relationship with his Qi counterpart, with both promising to recognize the other as "king". + + +Chu conquers Yue (334 BC) + +Early in the Warring States period, Chu was one of the strongest states in China. The state rose to a new level of power around 389 BC when King Dao of Chu (楚悼王) named the famous reformer Wu Qi as his chancellor. + + +Chu rose to its peak in 334 BC, when it conquered Yue to its east on the Pacific coast. The series of events leading up to this began when Yue prepared to attack Qi to its north. The King of Qi sent an emissary who persuaded the King of Yue to attack Chu instead. Yue initiated a large-scale attack at Chu but was defeated by Chu's counter-attack. Chu then proceeded to conquer Yue. + + +Qin, Han and Yan became kingdoms (325–323 BC) + +King Xian of Zhou had attempted to use what little royal prerogative he had left by appointing the dukes Xian (384–362 BC), Xiao (361–338 BC) and Hui (338–311 BC) of Qin as hegemons, thereby in theory making Qin the chief ally of the court. + + +However, in 325 the confidence of Duke Hui grew so great that he proclaimed himself "king" of Qin; adopting the same title as the king of Zhou and thereby effectively proclaiming independence from the Zhou dynasty. King Hui of Qin was guided by his prime minister Zhang Yi, a prominent representative of the School of Diplomacy. + + +He was followed in 323 BC by King Xuanhui of Han and King Yi of Yan, as well as King Cuo of the minor state Zhongshan. In 318 BC even the ruler of Song, a relatively minor state, declared himself king. Uniquely, while King Wuling of Zhao had joined the other kings in declaring himself king, he retracted this order in 318 BC, after Zhao suffered a great defeat at the hands of Qin. + + +Partition of Zhou (314 BC) + +King Kao of Zhou had enfeoffed his younger brother as Duke Huan of Henan. Three generations later, this cadet branch of the royal house began calling themselves "dukes of East Zhou". + + +Upon the ascension of King Nan in 314, East Zhou became an independent state. The king came to reside in what became known as West Zhou. + + +Horizontal and vertical alliances (334–249 BC) + +Towards the end of the Warring States period, the state of Qin became disproportionately powerful compared with the other six states. As a result, the policies of the six states became overwhelmingly oriented towards dealing with the Qin threat, with two opposing schools of thought. One school advocated a 'vertical' or north–south alliance called hezong (合縱) in which the states would ally with each other to repel Qin. The other advocated a 'horizontal' or east–west alliance called lianheng (連橫{), in which a state would ally with Qin to participate in its ascendancy. + + +There were some initial successes in hezong, though mutual suspicions between allied states led to the breakdown of such alliances. Qin repeatedly exploited the horizontal alliance strategy to defeat the states one by one. During this period, many philosophers and tacticians travelled around the states, recommending that the rulers put their respective ideas into use. These "lobbyists", such as Su Qin, who advocated vertical alliances, and Zhang Yi, who advocated horizontal alliances, were famous for their tact and intellect, and were collectively known as the School of Diplomacy, whose Chinese name (縱橫家 'the school of the vertical and horizontal') was derived from the two opposing ideas. + + +Su Qin and the first vertical alliance (334–300 BC) + +Beginning in 334 BC the diplomat Su Qin spent years visiting the courts of Yan, Zhao, Han, Wei, Qi and Chu and persuaded them to form a united front against Qin. In 318 BC all states except Qi launched a joint attack on Qin, which was not successful. + + +King Hui of Qin died in 311 BC, followed by prime minister Zhang Yi one year later. The new monarch, King Wu, reigned only four years before dying without legitimate heirs. Some damaging turbulence ensued throughout 307 BC before a son of King Hui by a concubine (i.e. a younger half-brother of King Wu) could be established as King Zhao, who in stark contrast to his predecessor went on to rule for an unprecedented 53 years. + + +After the failure of the first vertical alliance, Su Qin eventually came to live in Qi, where he was favored by King Xuan and drew the envy of the ministers. An assassination attempt in 300 BC left Su mortally wounded but not dead. Sensing death approaching, he advised the newly crowned King Min have him publicly executed to draw out the assassins. King Min complied with Su's request and killed him, putting an end to the first generation of Vertical alliance thinkers. + + +The first horizontal alliance (300–287 BC) + +King Min of Qi came to be highly influenced by Lord Mengchang, a grandson of the former King Wei of Qi. Lord Mengchang made a westward alliance with the states of Wei and Han. In the far west, Qin, which had been weakened by a succession struggle in 307, yielded to the new coalition and appointed Lord Mengchang its chief minister. The alliance between Qin and Qi was sealed by a Qin princess marrying King Min. This horizontal or east–west alliance might have secured peace except that it excluded the State of Zhao. + + +Around 299 BC, the ruler of Zhao became the last of the seven major states to proclaim himself "king". + + +In 298 BC, Zhao offered Qin an alliance and Lord Mengchang was driven out of Qin. The remaining three allies, Qi, Wei and Han, attacked Qin, driving up the Yellow River below Shanxi to the Hangu Pass. After 3 years of fighting they took the pass and forced Qin to return territory to Han and Wei. They next inflicted major defeats on Yan and Chu. During the 5-year administration of Lord Mengchang, Qi was the major power in China. + + +In 294, Lord Mengchang was implicated in a coup d'état and fled to Wei. His alliance system collapsed. +Qi and Qin made a truce and pursued their own interests. Qi moved south against the state of Song whilst the Qin General Bai Qi pushed back eastward against a Han/Wei alliance, gaining victory at the Battle of Yique. + + +In 288, King Zhao of Qin and King Min of Qi took the title di (帝 'emperor'), of the west and east respectively. They swore a covenant and started planning an attack on Zhao. + + +Su Dai and the second vertical alliance + +In 287 BC the strategist Su Dai, younger brother of Su Qin and possibly an agent of Yan, persuaded King Min that the Zhao war would only benefit Qin. King Min agreed and formed a 'vertical' alliance with the other states against Qin. Qin backed off, abandoned the presumptuous title of "Di", and restored territory to Wei and Zhao. In 286 Qi annexed the state of Song. + + +The second horizontal alliance and fall of Qi + +In 285 BC, the success of Qi had frightened the other states. Under the leadership of Lord Mengchang, who was exiled in Wei, Qin, Zhao, Wei and Yan formed an alliance. Yan had normally been a relatively weak ally of Qi and Qi feared little from this quarter. Yan's onslaught under general Yue Yi came as a devastating surprise. Simultaneously, the other allies attacked from the west. Chu declared itself an ally of Qi but contented itself with annexing some territory to its north. Qi's armies were destroyed while the territory of Qi was reduced to the two cities of Ju and Jimo. King Min himself was later captured and executed by his own followers. + + +King Min was succeeded by King Xiang in 283 BC. His general Tian Dan was eventually able to restore much of Qi's territory, but it never regained the influence it had under King Min. + + +Qin and Zhao expansion + +In 278 BC, the Qin general Bai Qi attacked from Qin's new territory in Sichuan to the west of Chu. The capital of Ying was captured and Chu's western lands on the Han River were lost. The effect was to shift Chu significantly to the east. + + +After Chu was defeated in 278, the remaining great powers were Qin in the west and Zhao in the north-center. There was little room for diplomatic maneuver and matters were decided by wars. Zhao had been much strengthened by King Wuling of Zhao (325–299). In 307 he enlarged his cavalry by copying the northern nomads. In 306 he took more land in the northern Shanxi plateau. In 305 he defeated the north-eastern border state of Zhongshan. In 304 he pushed far to the north-west and occupied the east–west section of the Yellow River in the north of the Ordos Loop. King Huiwen of Zhao (298–266) chose able servants and expanded against the weakened Qi and Wei. In 296 his general Lian Po defeated two Qin armies. + + +In 269 BC Fan Sui became chief advisor to Qin. He advocated authoritarian reforms, irrevocable expansion and an alliance with distant states to attack nearby states (the twenty-third of the Thirty-Six Stratagems). His maxim "attack not only the territory, but also the people" enunciated a policy of mass slaughter that became increasingly frequent. + + +Qin-Zhao wars (282–257 BC) + +In 265 King Zhaoxiang of Qin made the first move by attacking the weak state of Han which held the Yellow River gateway into Qin. He moved north-east across Wei territory to cut off the Han exclave of Shangdang north of Luoyang and south of Zhao. The Han king agreed to surrender Shangdang, but the local governor refused and presented it to King Xiaocheng of Zhao. Zhao sent out Lian Po who based his armies at Changping and Qin sent out general Wang He. Lian Po was too wise to risk a decisive battle with the Qin army and remained inside his fortifications. Qin could not break through and the armies were locked in stalemate for three years. The Zhao king decided that Lian Po was not aggressive enough and sent out Zhao Kuo who promised a decisive battle. At the same time Qin secretly replaced Wang He with the notoriously violent Bai Qi. When Zhao Kuo left his fortifications, Bai Qi used a Cannae maneuver, falling back in the center and surrounding the Zhao army from the sides. After being surrounded for 46 days, the starving Zhao troops surrendered in September 260 BC. It is said that Bai Qi had all the prisoners killed and that Zhao lost 400,000 men. + + +Qin was too exhausted to follow up its victory. Some time later it sent an army to besiege the Zhao capital but the army was destroyed when it was attacked from the rear. Zhao survived, but there was no longer a state that could resist Qin on its own. The other states could have survived if they remained united against Qin, but they did not. + + +In 257 BC, Qin army failed to besiege Handan and was defeated by the allied force of Zhao, Wei and Chu during the Battle of Handan. + + +End of Zhou dynasty (256–249 BC) + +The forces of King Zhao of Qin defeated King Nan of Zhou and conquered West Zhou in 256 BC, claiming the Nine Cauldrons and thereby symbolically becoming The Son of Heaven. + + +King Zhao's exceptionally long reign ended in 251 BC. His son King Xiaowen, already an old man, died just three days after his coronation and was succeeded by his son King Zhuangxiang of Qin. The new Qin king proceeded to conquer East Zhou, seven years after the fall of West Zhou. Thus the 800-year Zhou dynasty, nominally China's longest-ruling regime, finally came to an end. + + +Sima Qian contradicts himself regarding the ultimate fate of the East Zhou court. Chapter 4 (The Annals of Zhou) concludes with the sentence "thus the sacrifices of Zhou ended", but in the following chapter 5 (The Annals of Qin) we learn that "Qin did not prohibit their sacrifices; the Lord of Zhou was allotted a patch of land in Yangren where he could continue his ancestral sacrifices". + + +Qin unites China (247–221 BC) + +King Zhuangxiang of Qin ruled for only three years. He was succeeded by his son Zheng, who unlike the two elderly kings that preceded him was only 13 years old at his coronation. As an adult, Zheng became a brilliant commander who, in the span of just nine years, unified China. + + +Conquest of Han + +In 230 BC, Qin conquered Han. Han, the weakest of the Seven Warring States, was adjacent to the much stronger Qin, and had suffered continuous assaults by Qin in earlier years of the Warring States period. This went on until Emperor Qin Shi Huang sent general Wang Jian to attack Zhao. King An of Han, frightened by the thought that Han would be the next target of the Qin state, immediately sent diplomats to surrender the entire kingdom without a fight, saving the Han populace from the terrible potential consequences of an unsuccessful resistance. + + +Conquest of Wei + +In 225 BC, Qin conquered Wei. The Qin army led a direct invasion into Wei by besieging its capital Daliang but soon realized that the city walls were too tough to break into. They devised a new strategy in which they utilized the power of a local river that was linked to the Yellow River. The river was used to flood the city's walls, causing massive devastation to the city. Upon realizing the situation, King Jia of Wei hurriedly came out of the capital and surrendered it to the Qin army in order to avoid further bloodshed of his people. + + +Conquest of Chu + +In 223 BC, Qin conquered Chu. +The first invasion was however an utter disaster when 200,000 Qin troops, led by the general, Li Xin, were defeated by 500,000 Chu troops in the unfamiliar territory of Huaiyang, modern-day northern Jiangsu and Anhui provinces. Xiang Yan, the Chu commander, had lured Qin by allowing a few initial victories, but then counterattacked and burnt two large Qin camps. + + +In 222 BC, Wang Jian was recalled to lead a second military invasion with 600,000 men against the Chu state. High in morale after their victory in the previous year, the Chu forces were content to sit back and defend against what they expected to be a siege of Chu. However, Wang Jian decided to weaken Chu's resolve and tricked the Chu army by appearing to be idle in his fortifications whilst secretly training his troops to fight in Chu territory. After a year, the Chu defenders decided to disband due to apparent lack of action from the Qin. Wang Jian invaded at that point, with full force, and overran Huaiyang and the remaining Chu forces. Chu lost the initiative and could only sustain local guerrilla-style resistance until it too was fully conquered with the destruction of Shouchun and the death of its last leader, Lord Changping, in 223 BC. At their peak, the combined armies of Chu and Qin are estimated to have ranged from hundreds of thousands to a million soldiers, more than those involved in the campaign of Changping between Qin and Zhao 35 years earlier. + + +Conquest of Zhao and Yan + +In 222 BC, Qin conquered Zhao and Yan. +After the conquest of Zhao, the Qin army turned its attention towards Yan. Realizing the danger and gravity of this situation, Crown Prince Dan of Yan had sent Jing Ke to assassinate King Zheng of Qin, but this failure only helped to fuel the rage and determination of the Qin king, and he increased the number of troops to conquer the Yan state. + + +Conquest of Qi + +In 221 BC, Qin conquered Qi, the final unconquered state. It had not previously contributed or helped other states when Qin was conquering them. As soon as Qin's intention to invade it became clear, Qi swiftly surrendered all its cities, completing the unification of China and ushering in the Qin dynasty. The last Qi king lived out his days in exile in Gong and was not given a posthumous name, therefore he is known to posterity by his personal name Jian. + + +Aftermath + +The Qin king Ying Zheng declared himself as Qin Shi Huangdi, "The first Sovereign Emperor of Qin". + + +In the rule of the Qin state, the union was based solely on military power. The feudal holdings were abolished, and noble families were forced to live in the capital city Xianyang, in order to be supervised. A national road (as well as greater use of canals) allowed for faster and easier deployment and supply of the army. The peasants were given a wider range of land rights, although they were subject to taxation, creating a large amount of revenue to the state. + + +Military theory and practice + +Increasing scale of warfare + +The chariot remained a major factor in Chinese warfare long after it went out of fashion in the Middle East. Near the beginning of the Warring States period there is a shift from chariots to massed infantry, possibly associated with the invention of the crossbow. This had two major effects. First it led the dukes to weaken their chariot-riding nobility so they could get direct access to the peasantry who could be drafted as infantry. This change was associated with the shift from aristocratic to bureaucratic government. Second, it led to a massive increase in the scale of warfare. When the Zhou overthrew the Shang at the Battle of Muye they used 45,000 troops and 300 chariots. For the Warring States period the following figures for the military strengths of various states are reported: + + +For major battles, the following figures are reported: + + +Many scholars think these numbers are exaggerated (records are inadequate, they are much larger than those from similar societies, soldiers were paid by the number of enemies they killed and the Han dynasty had an interest in exaggerating the bloodiness of the age before China was unified). Regardless of exaggeration, it seems clear that warfare had become excessive during this period. The bloodshed and misery of the Warring States period goes a long way in explaining China's traditional and current preference for a united throne. + + +Military developments + +The Warring States period saw the introduction of many innovations to the art of warfare in China, such as the use of iron and of cavalry. + + +Warfare in the Warring States period evolved considerably from the Spring and Autumn period, as most armies made use of infantry and cavalry in battles, and the use of chariots became less widespread. The use of massed infantry made warfare bloodier and reduced the importance of the aristocracy, which in turn made the kings more despotic. From this period onward, as the various states competed with each other by mobilizing their armies to war, nobles in China belonged to the literate class, rather than to the warrior class as had previously been the case. + + +The various states fielded massive armies of infantry, cavalry, and chariots. Complex logistical systems maintained by efficient government bureaucracies were needed to supply, train, and control such large forces. The size of the armies ranged from tens of thousands to several hundred thousand men. +Iron weapons became more widespread and began to replace bronze. Most armour and weapons of this period were made from iron. + + +The first official native Chinese cavalry unit was formed in 307 BC during the military reforms of King Wuling of Zhao, who advocated 'nomadic dress and horse archery'. But the war chariot still retained its prestige and importance, despite the tactical superiority of cavalry. + + +The crossbow was the preferred long-range weapon of this period, due to several reasons. The crossbow could be mass-produced easily, and mass training of crossbowmen was possible. These qualities made it a powerful weapon against the enemy. + + +Infantrymen deployed a variety of weapons, but the most popular was the dagger-axe. The dagger-axe came in various lengths, from 9 to 18 feet; the weapon consisted of a thrusting spear with a slashing blade appended to it. Dagger-axes were an extremely popular weapon in various kingdoms, especially for the Qin, who produced 18-foot-long pike-like weapons. + + +The Qiang battle spear was named as the king 'wang' of all ancient weapons. It had the biggest impact on the battlefield and was quite difficult to master. The second important weapon of that era was the double-edged battle sword Jian. The fighting methods of using the Qiang spear and Jian sword were very different from what we see in movies or re-enactment shows today. Professional warriors of that era used the military concepts of "Master" Sun Tzu and created several successful "Ge Dou" martial schools. + + +Military thought + +The Warring States was a great period for military strategy; of the Seven Military Classics of China, four were written during this period: + + +Culture and society + +The Warring States period was an era of warfare in ancient China, as well as bureaucratic and military reforms and consolidation; the major states, ruling over large territories, quickly sought to consolidate their powers, leading to the final erosion of the Zhou court's prestige. As a sign of this shift, the rulers of all the major states (except for Chu, which had claimed kingly title much earlier) abandoned their former feudal titles for the title of 王, or King, claiming equality with the rulers of the Zhou. + + +At the same time, the constant conflict and need for innovative social and political models led to the development of many philosophical doctrines, later known as the Hundred Schools of Thought. The most notable schools of thought include Mohism (expounded by Mozi), Confucianism (represented by Mencius and Xunzi), Legalism (represented by Shang Yang, Shen Buhai, Shen Dao and Han Fei) and Taoism (represented by Zhuangzi and Lao Tzu). + + +The many states that were competing between each other attempted to display their power not only militarily but in their courts and in state philosophy. Many differing rulers adopted the differing philosophies to their own advantage or that of their kingdom. + + +Mencius attempted to instate Confucianism as a state philosophy, proposing that through the governing of moral principles like benevolence and righteousness, the state would win popular support from one state and those neighboring, eliminating the need of a war altogether. Mencius had attempted to convince King Hui of Liang, although was unsuccessful since the king saw no advantage in the period of wars. + + +Mohism was developed by Mozi (468–376 BC) and it provided a unified moral and political philosophy based on impartiality and benevolence. Mohists had the belief that people change depending on environments around. The same was applied to rulers, which is why one must be cautious of foreign influences. Mozi was very much against warfare, although he was a great tactician in defense. He defended the small state of Song from many attempts of the Chu state. + + +Taoism was advocated by Laozi, and believed that human nature was good and can achieve perfection by returning to its original state. It believed that like a baby, humans are simple and innocent although with development of civilizations it lost its innocence only to be replaced by fraud and greed. Contrarily to other schools, it did not want to gain influence in the offices of states and Laozi even refused to be the minister of the state of Chu. + + +Legalism created by Shang Yang in 338 BC, rejected all notions of religion and practices, and believed a nation should be governed by strict law. Not only were severe punishments applied, but they would be grouped with the families and made mutually responsible for criminal act. It proposed radical reforms, and established a society based on solid ranks. Peasants were encouraged to practice agriculture as occupation, and military performance was rewarded. Laws were also applied to all ranks with no exception; even the king was not above punishment. The philosophy was adapted by the Qin state and it created it into an organized, centralized state with a bureaucracy chosen on the basis of merit. +This period is most famous for the establishment of complex bureaucracies and centralized governments, as well as a clear legal system. The developments in political and military organization were the basis of the power of the Qin state, which conquered the other states and unified them under the Qin dynasty in 221 BC. + + +Nobles, bureaucrats and reformers + +The phenomenon of intensive warfare, based on mass formations of infantry rather than the traditional chariots, was one major trend which led to the creation of strong central bureaucracies in each of the major states. At the same time, the process of secondary feudalism which permeated the Spring and Autumn period, and led to such events as the partition of Jin and the usurpation of Qi by the Tian clan, was eventually reversed by the same process of bureaucratisation. + + +Under the demands of warfare, the states adopted bureaucratic reforms in the Warring States period. Wei adopted these in 445 BC, Zhao in 403 BC, Chu in 390 BC, Han in 355 BC, Qi in 357 BC and Qin in 350 BC. Power was centralised by curbing the landed aristocrats and sinecures and creating a new hierarchy based on meritorious service to the state, which were drawn from the lower rungs of society. Systematic auditing and reporting systems, and fixed salaries for officials were created. + + +The reforms of Shang Yang in Qin, and of Wu Qi in Chu, both centred on increased centralisation, the suppression of the nobility, and a vastly increased scope of government based on Legalist ideals, which were necessary to mobilise the large armies of the period. + + +Sophisticated arithmetic + +A bundle of 21 bamboo slips from the Tsinghua collection dated to 305 BC are the world's earliest example of a two digit decimal multiplication table, indicating that sophisticated commercial arithmetic was already established during this period. + + +Rod numerals were used to represent both negative and positive integers, and rational numbers, a true positional number system, with a blank for zero dating back to the Warring States period. + + +The nine linked-rings puzzle, an advanced puzzle device which requires mathematical analysis to solve, was invented during the period. + + +Literature + +An important literary achievement of the Warring States period is the Zuo Commentary on the Spring and Autumn Annals, which summarizes the preceding Spring and Autumn period. The less famous work Guoyu is thought to be by the same author. + + +Many sayings of Spring and Autumn philosophers, which had previously been circulated orally, were put into writing in the Warring States. These include the Analects and The Art of War. + + +Economic developments + +The Warring States period saw the proliferation of iron working in China, replacing bronze as the dominant type of metal used in warfare. Areas such as Shu (present-day Sichuan) and Yue (present-day Zhejiang) were also brought into the Chinese cultural sphere during this time. Trade also became important, and some merchants had considerable power in politics, the most prominent of which was Lü Buwei, who rose to become Chancellor of Qin and was a key supporter of the eventual Qin Shihuang. + + +At the same time, the increased resources of consolidated, bureaucratic states, coupled with the logistical needs of mass levies and large-scale warfare, led to the proliferation of economic projects such as large-scale waterworks. Major examples of such waterworks include the Dujiangyan Irrigation System, which controlled the Min River in Sichuan and turned the former backwater region into a major Qin logistical base, and the Zhengguo Canal which irrigated large areas of land in the Guanzhong Plain, again increasing Qin's agricultural output. + + +The Guanzi is considered one of the most foundational texts of the developing political economy in the Warring States period. It addresses principles of price regulation in the context of effectively dealing with commodities that are "light" (connoting a commodity which is unimportant, non-essential, or inexpensive) or "heavy" (a commodity which is important, essential, or expensive) and how whether a commodity is "light" or "heavy" is understood in relation to other commodities. + + +In summary:""" + # fmt: on + + input_ids = tokenizer(LONG_TEXT, return_tensors="pt").input_ids.to(torch_device) + generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] + + torch.testing.assert_close(generated_ids.cpu(), torch.tensor([[ 576, 4570, 71818, 374, 6509, 825, 315, 279, 1429, 88228, 21984, 315, 279, 11220, 4948, 8584, 304, 279, 467, 19859, 4180, 4168, 13, 1084, 14230, 16170, 315, 3349, 19256, 304, 279, 2266, 315, 13444, 14550, 448, 50867, 429, 525, 330, 4145, 1, 476, 330, 88845, 1, 323, 1246, 3425, 264]], dtype=torch.long)) # fmt: skip + self.assertEqual( + tokenizer.decode(generated_ids[0]), + """ The Guanzi is considered one of the most foundational texts of the developing political economy in the Warring States period. It addresses principles of price regulation in the context of effectively dealing with commodities that are "light" or "heavy" and how whether a""", + ) + model.config._attn_implementation = "eager" + new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] + with self.subTest("Eager matches sdpa"): + torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) + + model.config._attn_implementation = "flex_attention" + new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] + with self.subTest("Eager matches Flex attention"): + torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) + + model.config._attn_implementation = "flash_attention_2" + new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] + with self.subTest("Eager matches flash attention"): + torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) From 72e87baa9689177557d92f2a0a809c0687c7ddf1 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 28 Feb 2025 16:00:36 +0000 Subject: [PATCH 17/48] Copy prepare_inputs_for_generation() from the in-library Phi3 class --- .../models/phi3_v/modeling_phi3_v.py | 75 +++++-------------- 1 file changed, 18 insertions(+), 57 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 1b6906bbe2c9..737112c27841 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -1637,10 +1637,15 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - pixel_values=None, - image_sizes=None, + cache_position=None, + position_ids=None, + use_cache=True, + logits_to_keep=None, **kwargs, ): + # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the + # process + # When the first time input length reached long and short factor switching point, enforce re-compute cache # It will cause downside of slower at this single token position, however, better than current failure. if ( @@ -1648,64 +1653,20 @@ def prepare_inputs_for_generation( and self.config.rope_scaling and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = ( - past_key_values.seen_tokens if isinstance(past_key_values, Cache) else past_key_values[0][0].shape[2] - ) + past_length = cache_position[0] if past_length <= self.config.original_max_position_embeddings: past_key_values = None - if past_key_values is not None: - if isinstance(past_key_values, Cache): - cache_length = past_key_values.get_seq_length() - past_length = past_key_values.seen_tokens - max_cache_length = past_key_values.get_max_cache_shape() - else: - cache_length = past_length = past_key_values[0][0].shape[2] - max_cache_length = None - - # Keep only the unprocessed tokens: - # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where - # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as - # input) - if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: - input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] - # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard - # input_ids based on the past_length. - elif past_length < input_ids.shape[1]: - input_ids = input_ids[:, past_length:] - # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. - - # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. - if ( - max_cache_length is not None - and attention_mask is not None - and cache_length + input_ids.shape[1] > max_cache_length - ): - attention_mask = attention_mask[:, -max_cache_length:] - - position_ids = kwargs.get("position_ids", None) - if attention_mask is not None and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_key_values: - position_ids = position_ids[:, -input_ids.shape[1] :] - - # if `inputs_embeds` are passed, we only want to use them in the 1st generation step - if inputs_embeds is not None and past_key_values is None: - model_inputs = {"inputs_embeds": inputs_embeds} - else: - model_inputs = {"input_ids": input_ids} - - model_inputs.update( - { - "position_ids": position_ids, - "past_key_values": past_key_values, - "use_cache": kwargs.get("use_cache"), - "attention_mask": attention_mask, - "pixel_values": pixel_values, - "image_sizes": image_sizes, - } + model_inputs = super().prepare_inputs_for_generation( + input_ids=input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + use_cache=use_cache, + logits_to_keep=logits_to_keep, + **kwargs, ) return model_inputs From 063817d50c2524028cf7efed0d95d7704024f94c Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 28 Feb 2025 16:09:53 +0000 Subject: [PATCH 18/48] break copied from because of a missing questionanswering class --- tests/models/phi3_v/test_modeling_phi3_v.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py index e644d31907b8..74de67922f65 100644 --- a/tests/models/phi3_v/test_modeling_phi3_v.py +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -298,7 +298,6 @@ def prepare_config_and_inputs_for_common(self): @require_torch -# Copied from tests.models.mistral.test_modeling_mistral.MistralModelTest with Mistral->Phi3V class Phi3VModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = ( ( From 7bc578465c20e69d9bf5cc63e62d3279a42a62e7 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 2 Mar 2025 16:56:45 -0700 Subject: [PATCH 19/48] added Phi3VIntegrationTest to test_modeling_phi3_v --- .../en/model_doc/{phi3_5.md => phi3_v.md} | 0 tests/models/phi3_v/test_modeling_phi3_v.py | 636 ++---------------- 2 files changed, 39 insertions(+), 597 deletions(-) rename docs/source/en/model_doc/{phi3_5.md => phi3_v.md} (100%) diff --git a/docs/source/en/model_doc/phi3_5.md b/docs/source/en/model_doc/phi3_v.md similarity index 100% rename from docs/source/en/model_doc/phi3_5.md rename to docs/source/en/model_doc/phi3_v.md diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py index 74de67922f65..411ffab3df0c 100644 --- a/tests/models/phi3_v/test_modeling_phi3_v.py +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -20,7 +20,7 @@ import pytest from packaging import version -from transformers import AutoTokenizer, Phi3VConfig, is_torch_available, set_seed +from transformers import AutoTokenizer, Phi3VConfig, is_torch_available, is_vision_available, set_seed from transformers.generation.configuration_utils import GenerationConfig from transformers.testing_utils import ( backend_empty_cache, @@ -30,7 +30,7 @@ require_torch_gpu, require_torch_sdpa, slow, - torch_device, + torch_device, require_vision, ) from ...generation.test_utils import GenerationTesterMixin @@ -47,9 +47,14 @@ Phi3VForSequenceClassification, Phi3VForTokenClassification, Phi3VModel, + Phi3VProcessor, ) +if is_vision_available(): + from PIL import Image + + class Phi3VModelTester: def __init__( self, @@ -429,609 +434,46 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): self.skipTest(reason="Phi3V flash attention does not support right padding") -@require_torch -class Phi3VIntegrationTest(unittest.TestCase): - @slow - def test_model_450m_logits(self): - input_ids = [1, 306, 4658, 278, 6593, 310, 2834, 338] - model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto") - input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) - with torch.no_grad(): - out = model(input_ids).logits.float().cpu() - # Expected mean on dim = -1 - EXPECTED_MEAN = torch.tensor([[-1.9537, -1.6193, -1.4123, -1.4673, -1.8511, -1.9309, -1.9826, -2.1776]]) - torch.testing.assert_close(out.mean(-1), EXPECTED_MEAN, rtol=1e-2, atol=1e-2) - # slicing logits[0, 0, 0:30] - EXPECTED_SLICE = torch.tensor([3.2025, 7.1265, 4.6058, 3.6423, 1.6357, 3.9265, 5.1883, 5.8760, 2.7942, 4.4823, 3.2571, 2.1063, 3.4275, 4.2028, 1.9767, 5.2115, 6.6756, 6.3999, 6.0483, 5.7378, 5.6660, 5.2298, 5.4103, 5.1248, 5.4376, 2.4570, 2.6107, 5.4039, 2.8077, 4.7777]) # fmt: skip - print(out[0, 0, :30]) - torch.testing.assert_close(out[0, 0, :30], EXPECTED_SLICE, rtol=1e-4, atol=1e-4) - - del model - backend_empty_cache(torch_device) - gc.collect() - - @slow - def test_model_450m_generation(self): - EXPECTED_TEXT_COMPLETION = ( - """My favourite condiment is 100% natural, organic and vegan. I love to use it in my cooking and I""" - ) - prompt = "My favourite condiment is " - tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-0.5B", use_fast=False) - model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto") - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) - - # greedy generation outputs - generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) - text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) - self.assertEqual(EXPECTED_TEXT_COMPLETION, text) +def get_images(): + images = [] + url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg" + images.append(Image.open(requests.get(url, stream=True).raw)) + return images - del model - backend_empty_cache(torch_device) - gc.collect() - @require_bitsandbytes - @slow - @require_flash_attn - @pytest.mark.flash_attn_test - def test_model_450m_long_prompt(self): - EXPECTED_OUTPUT_TOKEN_IDS = [306, 338] - # An input with 4097 tokens that is above the size of the sliding window - input_ids = [1] + [306, 338] * 2048 +@require_torch +@require_vision +@slow +class Phi3VIntegrationTest(unittest.TestCase): + def test_casual_generate(self): model = Phi3VForCausalLM.from_pretrained( - "Qwen/Phi3V-0.5B", - device_map="auto", - load_in_4bit=True, - attn_implementation="flash_attention_2", - ) - input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) - generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - # Assisted generation - assistant_model = model - assistant_model.generation_config.num_assistant_tokens = 2 - assistant_model.generation_config.num_assistant_tokens_schedule = "constant" - generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - del assistant_model - del model - backend_empty_cache(torch_device) - gc.collect() - - @slow - @require_torch_sdpa - def test_model_450m_long_prompt_sdpa(self): - EXPECTED_OUTPUT_TOKEN_IDS = [306, 338] - # An input with 4097 tokens that is above the size of the sliding window - input_ids = [1] + [306, 338] * 2048 - model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto", attn_implementation="sdpa") - input_ids = torch.tensor([input_ids]).to(model.model.embed_tokens.weight.device) - generated_ids = model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - # Assisted generation - assistant_model = model - assistant_model.generation_config.num_assistant_tokens = 2 - assistant_model.generation_config.num_assistant_tokens_schedule = "constant" - generated_ids = assistant_model.generate(input_ids, max_new_tokens=4, temperature=0) - self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-2:].tolist()) - - del assistant_model - - backend_empty_cache(torch_device) - gc.collect() - - EXPECTED_TEXT_COMPLETION = ( - "My favourite condiment is 100% natural, organic and vegan. I love to use it in my cooking and I" - ) - prompt = "My favourite condiment is " - tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-0.5B", use_fast=False) - - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) - - # greedy generation outputs - generated_ids = model.generate(input_ids, max_new_tokens=20, temperature=0) - text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) - self.assertEqual(EXPECTED_TEXT_COMPLETION, text) - - @slow - def test_speculative_generation(self): - EXPECTED_TEXT_COMPLETION = ( - "My favourite condiment is 100% natural honey, and I always like to use it in my recipes. I love" - ) - prompt = "My favourite condiment is " - tokenizer = AutoTokenizer.from_pretrained("Qwen/Phi3V-7B", use_fast=False) - model = Phi3VForCausalLM.from_pretrained("Qwen/Phi3V-0.5B", device_map="auto", torch_dtype=torch.float16) - assistant_model = Phi3VForCausalLM.from_pretrained( - "Qwen/Phi3V-0.5B", device_map="auto", torch_dtype=torch.float16 - ) - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.model.embed_tokens.weight.device) - - # greedy generation outputs - set_seed(0) - generated_ids = model.generate( - input_ids, max_new_tokens=20, do_sample=True, temperature=0.3, assistant_model=assistant_model - ) - text = tokenizer.decode(generated_ids[0], skip_special_tokens=True) - self.assertEqual(EXPECTED_TEXT_COMPLETION, text) - - del model - backend_empty_cache(torch_device) - gc.collect() - - @slow - def test_export_static_cache(self): - if version.parse(torch.__version__) < version.parse("2.4.0"): - self.skipTest(reason="This test requires torch >= 2.4 to run.") - - from transformers.integrations.executorch import ( - TorchExportableModuleWithStaticCache, - convert_and_export_with_cache, - ) - - qwen_model = "Qwen/Phi3V-0.5B" - - tokenizer = AutoTokenizer.from_pretrained(qwen_model, pad_token="", padding_side="right") - EXPECTED_TEXT_COMPLETION = [ - "My favourite condiment is 100% natural, organic, gluten free, vegan, and free from preservatives. I" + "microsoft/Phi-3.5-vision-instruct", + torch_dtype="auto", + _attn_implementation='flash_attention_2').to(torch_device) + processor = Phi3VProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", + num_crops=4) + images = get_images() + messages = [ + {"role": "user", "content": "<|image_1|>\nWhat in this image is in motion?"}, ] - max_generation_length = tokenizer(EXPECTED_TEXT_COMPLETION, return_tensors="pt", padding=True)[ - "input_ids" - ].shape[-1] - # Load model - device = "cpu" - dtype = torch.bfloat16 - cache_implementation = "static" - attn_implementation = "sdpa" - batch_size = 1 - model = Phi3VForCausalLM.from_pretrained( - qwen_model, - device_map=device, - torch_dtype=dtype, - attn_implementation=attn_implementation, - generation_config=GenerationConfig( - use_cache=True, - cache_implementation=cache_implementation, - max_length=max_generation_length, - cache_config={ - "batch_size": batch_size, - "max_cache_len": max_generation_length, - }, - ), + prompt = processor.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True ) + inputs = processor(prompt, images, return_tensors="pt").to(torch_device) - prompt = ["My favourite condiment is "] - prompt_tokens = tokenizer(prompt, return_tensors="pt", padding=True).to(model.device) - prompt_token_ids = prompt_tokens["input_ids"] - max_new_tokens = max_generation_length - prompt_token_ids.shape[-1] - - # Static Cache + export - exported_program = convert_and_export_with_cache(model) - ep_generated_ids = TorchExportableModuleWithStaticCache.generate( - exported_program=exported_program, prompt_token_ids=prompt_token_ids, max_new_tokens=max_new_tokens - ) - ep_generated_text = tokenizer.batch_decode(ep_generated_ids, skip_special_tokens=True) - self.assertEqual(EXPECTED_TEXT_COMPLETION, ep_generated_text) - - @require_flash_attn - @slow - def test_3b_generation(self): - model_id = "Qwen/Phi3V.5-3B" - tokenizer = AutoTokenizer.from_pretrained(model_id) - model = Phi3VForCausalLM.from_pretrained( - model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048 - ).to(torch_device) - # we need a long text to test sliding window - # fmt: off - LONG_TEXT = """The Warring States period in Chinese history (c. 475 – 221 BC) comprises the final centuries of the Zhou dynasty (c. 1046 – 256 BC), which were characterized by warfare, bureaucratic and military reform, and political consolidation. It followed the Spring and Autumn period and concluded with the wars of conquest that saw the state of Qin annex each of the other contender states by 221 BC and found the Qin dynasty, the first imperial dynastic state in East Asian history. - - -While scholars have identified several different dates as marking the beginning of the Warring States period, Sima Qian's choice of 475 BC is the most often cited. The era largely corresponds to the second half of the Eastern Zhou period, where the king of Zhou formally ruled as Chinese sovereign, but had lost political power and functioned in practice as a figurehead. This dynamic served as the backdrop for the machinations of the eponymous Warring States. The label "Warring States period" derives from the Record of the Warring States, a work of history compiled during the early Han dynasty (202 BC – 220 AD). - - -Geography - -The political geography of the era was dominated by the Seven Warring States, namely: - - -Besides these seven major states other smaller states survived into the period. They include: - - -Periodisation - -The eastward flight of the Zhou court in 771 BC marks the start of the Spring and Autumn period. No one single incident or starting point inaugurated the Warring States era. The political situation of the period represented a culmination of historical trends of conquest and annexation which also characterised the Spring and Autumn period. As a result, there is some controversy as to the beginning of the era. Proposed starting points include: - - -History - -Background and formation - -The Eastern Zhou dynasty began its fall around 5th century BC. As their influence waned, they had to rely on armies in allied states rather than their own military force. Hundreds of smaller polities coalesced into seven major states which included: Chu, Han, Qin, Wei, Yan, Qi and Zhao. However, there eventually was a shift in alliances because each state's ruler wanted independence. This caused hundreds of wars between 535 and 286 BC. The victorious state would have overall rule and control in China. - - -The system of feudal states created by the Western Zhou dynasty underwent enormous changes after 771 BC with the flight of the Zhou court to modern-day Luoyang and the diminution of its relevance and power. The Spring and Autumn period led to a few states gaining power at the expense of many others, the latter no longer able to depend on central authority for legitimacy or protection. During the Warring States period, many rulers claimed the Mandate of Heaven to justify their conquest of other states and spread their influence. - - -The struggle for hegemony eventually created a state system dominated by several large states, such as Jin, Chu, Qin, Yan, and Qi, while the smaller states of the Central Plain tended to be their satellites and tributaries. Other major states also existed, such as Wu and Yue in the southeast. The last decades of the Spring and Autumn era were marked by increased stability, as the result of peace negotiations between Jin and Chu which established their respective spheres of influence. This situation ended with the partition of Jin, whereby the state was divided between the houses of Han, Zhao and Wei, leading to the seven major warring states. - - -Partition of Jin (453–403 BC) - -The rulers of Jin had steadily lost political powers since the middle of the 6th century BC to their nominally subordinate nobles and military commanders, a situation arising from the traditions of the Jin which forbade the enfeoffment of relatives of the ducal house. This allowed other clans to gain fiefs and military authority, and decades of internecine struggle led to the establishment of four major families, the Han, Zhao, Wei and Zhi. - - -The Battle of Jinyang saw the allied Han, Zhao and Wei destroy the Zhi family (453 BC) and their lands were distributed among them. With this, they became the de facto rulers of most of Jin's territory, though this situation would not be officially recognised until half a century later. The Jin division created a political vacuum that enabled during the first 50 years expansion of Chu and Yue northward and Qi southward. Qin increased its control of the local tribes and began its expansion southwest to Sichuan. - - -Early Warring States - -The three Jins recognized (403–364 BC) - -In 403 BC, the court of King Weilie of Zhou officially recognized Zhao, Wei and Han as immediate vassals, thereby raising them to the same rank as the other warring states. - - -From before 405 until 383 BC the three Jins were united under the leadership of Wei and expanded in all directions. The most important figure was Marquess Wen of Wei (445–396 BC). In 408–406 BC he conquered the State of Zhongshan to the northeast on the other side of Zhao. At the same time he pushed west across the Yellow River to the Luo River taking the area of Xihe (literally 'west of the river'). - - -The growing power of Wei caused Zhao to back away from the alliance. In 383 BC it moved its capital to Handan and attacked the small state of Wey. Wey appealed to Wei which attacked Zhao on the western side. Being in danger, Zhao called in Chu. As usual, Chu used this as a pretext to annex territory to its north, but the diversion allowed Zhao to occupy a part of Wei. This conflict marked the end of the power of the united Jins and the beginning a period of shifting alliances and wars on several fronts. - - -In 376 BC, the states of Han, Wei and Zhao deposed Duke Jing of Jin and divided the last remaining Jin territory between themselves, which marked the final end of the Jin state. - - -In 370 BC, Marquess Wu of Wei died without naming a successor, which led to a war of succession. After three years of civil war, Zhao from the north and Han from the south invaded Wei. On the verge of conquering Wei, the leaders of Zhao and Han fell into disagreement about what to do with Wei, and both armies abruptly retreated. As a result, King Hui of Wei (still a Marquess at the time) was able to ascend the throne of Wei. - - -Zhao extended from the Shanxi plateau across the plain to the borders of Qi. Wei reached east to Qi, Lu, and Song. To the south, the weaker state of Han held the east–west part of the Yellow River valley, surrounded the Zhou royal domain at Luoyang and held an area north of Luoyang called Shangdang. - - -Qi resurgence under Tian (379–340 BC) - -Duke Kang of Qi died in 379 BC with no heir from the house of Jiang, which had ruled Qi since the state's founding. The throne instead passed to the future King Wei, from the house of Tian. The Tian had been very influential at court towards the end of Jiang rule, and now openly assumed power. - - -The new ruler set about reclaiming territories that had been lost to other states. He launched a successful campaign against Zhao, Wey and Wei, once again extending Qi territory to the Great Wall. Sima Qian writes that the other states were so awestruck that nobody dared attack Qi for more than 20 years. The demonstrated military prowess also had a calming effect on Qi's own population, which experienced great domestic tranquility during Wei's reign. - - -By the end of King Wei's reign, Qi had become the strongest of the states and proclaimed itself "king"; establishing independence from the Zhou dynasty (see below). - - -Wars of Wei - -King Hui of Wei (370–319 BC) set about restoring the state. In 362–359 BC he exchanged territories with Han and Zhao in order to make the boundaries of the three states more rational. - - -In 364 BC, Wei was defeated by Qin at the Battle of Shimen and was only saved by the intervention of Zhao. Qin won another victory in 362 BC. In 361 BC the Wei capital was moved east to Daliang to be out of the reach of Qin. - - -In 354 BC, King Hui of Wei started a large-scale attack on Zhao. By 353 BC, Zhao was losing badly and its capital, Handan, was under siege. The state of Qi intervened. The famous Qi strategist, Sun Bin the great-great-great-grandson of Sun Tzu, the author of the Art of War, proposed to attack the Wei capital while the Wei army was tied up besieging Zhao. The strategy was a success; the Wei army hastily moved south to protect its capital, was caught on the road and decisively defeated at the Battle of Guiling. The battle is remembered in the second of the Thirty-Six Stratagems, "besiege Wei, save Zhao"—meaning to attack a vulnerable spot to relieve pressure at another point. - - -Domestically, King Hui patronized philosophy and the arts, and is perhaps best remembered for hosting the Confucian philosopher Mencius at his court; their conversations form the first two chapters of the book which bears Meng Zi's name. - - -Dukes become kings - -Qi and Wei became kingdoms (344 BC) - -The title of king (wang, 王) was held by figurehead rulers of the Zhou dynasty, while the rulers of most states held the title of duke (gong, 公) or marquess (hou, 侯). A major exception was Chu, whose rulers were called kings since King Wu of Chu started using the title c. 703 BC. - - -In 344 BC the rulers of Qi and Wei mutually recognized each other as kings: King Wei of Qi and King Hui of Wei, in effect declaring their independence from the Zhou court. This marked a major turning point: unlike those in the Spring and Autumn period, the new generation of rulers ascending the thrones in the Warring States period would not entertain even the pretence of being vassals of the Zhou dynasty, instead proclaiming themselves fully independent kingdoms. - - -Shang Yang reforms Qin (356–338 BC) - -During the early Warring States period Qin generally avoided conflicts with the other states. This changed during the reign of Duke Xiao, when prime minister Shang Yang made centralizing and authoritarian reforms in accordance with his Legalist philosophy between the years 356 and 338 BC. - - -Shang introduced land reforms, privatized land, rewarded farmers who exceeded harvest quotas, enslaved farmers who failed to meet quotas, and used enslaved subjects as rewards for those who met government policies. As manpower was short in Qin relative to the other states at the time, Shang enacted policies to increase its manpower. As Qin peasants were recruited into the military, he encouraged active immigration of peasants from other states into Qin as a replacement workforce; this policy simultaneously increased the manpower of Qin and weakened the manpower of Qin's rivals. - - -Shang made laws forcing citizens to marry at a young age and passed tax laws to encourage raising multiple children. He also enacted policies to free convicts who worked in opening wastelands for agriculture. Shang abolished primogeniture and created a double tax on households that had more than one son living in the household, to break up large clans into nuclear families. Shang also moved the capital to reduce the influence of nobles on the administration. - - -The rise of Qin was recognized by the royal court, and in 343 BC the king conferred the title of Count (伯 Bó) on Duke Xiao. As was customary, a conference was hosted which the feudal lords attended, and during which the Son of Heaven bestowed the title. - - -After the reforms Qin became much more aggressive. In 340 Qin took land from Wèi after it had been defeated by Qi. In 316 Qin conquered Shu and Ba in Sichuan to the southwest. Development of this area took a long time but slowly added greatly to Qin's wealth and power. - - -Qin defeats Wei (341–340 BC) - -In 341 BC, Wei attacked Han. Qi allowed Han to be nearly defeated and then intervened. The generals from the Battle of Guiling met again (Sun Bin and Tian Ji versus Pang Juan), using the same tactic, attacking Wei's capital. Sun Bin feigned a retreat and then turned on the overconfident Wei troops and decisively defeated them at the Battle of Maling. After the battle all three of the Jin successor states appeared before King Xuan of Qi, pledging their loyalty. - - -In the following year Qin attacked the weakened Wei. Wei was devastatingly defeated and ceded a large part of its territory in return for truce. With Wei severely weakened, Qi and Qin became the dominant states in China. - - -Wei came to rely on Qi for protection, with King Hui of Wei meeting King Xuan of Qi on two occasions. After Hui's death, his successor King Xiang also established a good relationship with his Qi counterpart, with both promising to recognize the other as "king". - - -Chu conquers Yue (334 BC) - -Early in the Warring States period, Chu was one of the strongest states in China. The state rose to a new level of power around 389 BC when King Dao of Chu (楚悼王) named the famous reformer Wu Qi as his chancellor. - - -Chu rose to its peak in 334 BC, when it conquered Yue to its east on the Pacific coast. The series of events leading up to this began when Yue prepared to attack Qi to its north. The King of Qi sent an emissary who persuaded the King of Yue to attack Chu instead. Yue initiated a large-scale attack at Chu but was defeated by Chu's counter-attack. Chu then proceeded to conquer Yue. - - -Qin, Han and Yan became kingdoms (325–323 BC) - -King Xian of Zhou had attempted to use what little royal prerogative he had left by appointing the dukes Xian (384–362 BC), Xiao (361–338 BC) and Hui (338–311 BC) of Qin as hegemons, thereby in theory making Qin the chief ally of the court. - - -However, in 325 the confidence of Duke Hui grew so great that he proclaimed himself "king" of Qin; adopting the same title as the king of Zhou and thereby effectively proclaiming independence from the Zhou dynasty. King Hui of Qin was guided by his prime minister Zhang Yi, a prominent representative of the School of Diplomacy. - - -He was followed in 323 BC by King Xuanhui of Han and King Yi of Yan, as well as King Cuo of the minor state Zhongshan. In 318 BC even the ruler of Song, a relatively minor state, declared himself king. Uniquely, while King Wuling of Zhao had joined the other kings in declaring himself king, he retracted this order in 318 BC, after Zhao suffered a great defeat at the hands of Qin. - - -Partition of Zhou (314 BC) - -King Kao of Zhou had enfeoffed his younger brother as Duke Huan of Henan. Three generations later, this cadet branch of the royal house began calling themselves "dukes of East Zhou". - - -Upon the ascension of King Nan in 314, East Zhou became an independent state. The king came to reside in what became known as West Zhou. - - -Horizontal and vertical alliances (334–249 BC) - -Towards the end of the Warring States period, the state of Qin became disproportionately powerful compared with the other six states. As a result, the policies of the six states became overwhelmingly oriented towards dealing with the Qin threat, with two opposing schools of thought. One school advocated a 'vertical' or north–south alliance called hezong (合縱) in which the states would ally with each other to repel Qin. The other advocated a 'horizontal' or east–west alliance called lianheng (連橫{), in which a state would ally with Qin to participate in its ascendancy. - - -There were some initial successes in hezong, though mutual suspicions between allied states led to the breakdown of such alliances. Qin repeatedly exploited the horizontal alliance strategy to defeat the states one by one. During this period, many philosophers and tacticians travelled around the states, recommending that the rulers put their respective ideas into use. These "lobbyists", such as Su Qin, who advocated vertical alliances, and Zhang Yi, who advocated horizontal alliances, were famous for their tact and intellect, and were collectively known as the School of Diplomacy, whose Chinese name (縱橫家 'the school of the vertical and horizontal') was derived from the two opposing ideas. - - -Su Qin and the first vertical alliance (334–300 BC) - -Beginning in 334 BC the diplomat Su Qin spent years visiting the courts of Yan, Zhao, Han, Wei, Qi and Chu and persuaded them to form a united front against Qin. In 318 BC all states except Qi launched a joint attack on Qin, which was not successful. - - -King Hui of Qin died in 311 BC, followed by prime minister Zhang Yi one year later. The new monarch, King Wu, reigned only four years before dying without legitimate heirs. Some damaging turbulence ensued throughout 307 BC before a son of King Hui by a concubine (i.e. a younger half-brother of King Wu) could be established as King Zhao, who in stark contrast to his predecessor went on to rule for an unprecedented 53 years. - - -After the failure of the first vertical alliance, Su Qin eventually came to live in Qi, where he was favored by King Xuan and drew the envy of the ministers. An assassination attempt in 300 BC left Su mortally wounded but not dead. Sensing death approaching, he advised the newly crowned King Min have him publicly executed to draw out the assassins. King Min complied with Su's request and killed him, putting an end to the first generation of Vertical alliance thinkers. - - -The first horizontal alliance (300–287 BC) - -King Min of Qi came to be highly influenced by Lord Mengchang, a grandson of the former King Wei of Qi. Lord Mengchang made a westward alliance with the states of Wei and Han. In the far west, Qin, which had been weakened by a succession struggle in 307, yielded to the new coalition and appointed Lord Mengchang its chief minister. The alliance between Qin and Qi was sealed by a Qin princess marrying King Min. This horizontal or east–west alliance might have secured peace except that it excluded the State of Zhao. - - -Around 299 BC, the ruler of Zhao became the last of the seven major states to proclaim himself "king". - - -In 298 BC, Zhao offered Qin an alliance and Lord Mengchang was driven out of Qin. The remaining three allies, Qi, Wei and Han, attacked Qin, driving up the Yellow River below Shanxi to the Hangu Pass. After 3 years of fighting they took the pass and forced Qin to return territory to Han and Wei. They next inflicted major defeats on Yan and Chu. During the 5-year administration of Lord Mengchang, Qi was the major power in China. - - -In 294, Lord Mengchang was implicated in a coup d'état and fled to Wei. His alliance system collapsed. -Qi and Qin made a truce and pursued their own interests. Qi moved south against the state of Song whilst the Qin General Bai Qi pushed back eastward against a Han/Wei alliance, gaining victory at the Battle of Yique. - - -In 288, King Zhao of Qin and King Min of Qi took the title di (帝 'emperor'), of the west and east respectively. They swore a covenant and started planning an attack on Zhao. - - -Su Dai and the second vertical alliance - -In 287 BC the strategist Su Dai, younger brother of Su Qin and possibly an agent of Yan, persuaded King Min that the Zhao war would only benefit Qin. King Min agreed and formed a 'vertical' alliance with the other states against Qin. Qin backed off, abandoned the presumptuous title of "Di", and restored territory to Wei and Zhao. In 286 Qi annexed the state of Song. - - -The second horizontal alliance and fall of Qi - -In 285 BC, the success of Qi had frightened the other states. Under the leadership of Lord Mengchang, who was exiled in Wei, Qin, Zhao, Wei and Yan formed an alliance. Yan had normally been a relatively weak ally of Qi and Qi feared little from this quarter. Yan's onslaught under general Yue Yi came as a devastating surprise. Simultaneously, the other allies attacked from the west. Chu declared itself an ally of Qi but contented itself with annexing some territory to its north. Qi's armies were destroyed while the territory of Qi was reduced to the two cities of Ju and Jimo. King Min himself was later captured and executed by his own followers. - - -King Min was succeeded by King Xiang in 283 BC. His general Tian Dan was eventually able to restore much of Qi's territory, but it never regained the influence it had under King Min. - - -Qin and Zhao expansion - -In 278 BC, the Qin general Bai Qi attacked from Qin's new territory in Sichuan to the west of Chu. The capital of Ying was captured and Chu's western lands on the Han River were lost. The effect was to shift Chu significantly to the east. - - -After Chu was defeated in 278, the remaining great powers were Qin in the west and Zhao in the north-center. There was little room for diplomatic maneuver and matters were decided by wars. Zhao had been much strengthened by King Wuling of Zhao (325–299). In 307 he enlarged his cavalry by copying the northern nomads. In 306 he took more land in the northern Shanxi plateau. In 305 he defeated the north-eastern border state of Zhongshan. In 304 he pushed far to the north-west and occupied the east–west section of the Yellow River in the north of the Ordos Loop. King Huiwen of Zhao (298–266) chose able servants and expanded against the weakened Qi and Wei. In 296 his general Lian Po defeated two Qin armies. - - -In 269 BC Fan Sui became chief advisor to Qin. He advocated authoritarian reforms, irrevocable expansion and an alliance with distant states to attack nearby states (the twenty-third of the Thirty-Six Stratagems). His maxim "attack not only the territory, but also the people" enunciated a policy of mass slaughter that became increasingly frequent. - - -Qin-Zhao wars (282–257 BC) - -In 265 King Zhaoxiang of Qin made the first move by attacking the weak state of Han which held the Yellow River gateway into Qin. He moved north-east across Wei territory to cut off the Han exclave of Shangdang north of Luoyang and south of Zhao. The Han king agreed to surrender Shangdang, but the local governor refused and presented it to King Xiaocheng of Zhao. Zhao sent out Lian Po who based his armies at Changping and Qin sent out general Wang He. Lian Po was too wise to risk a decisive battle with the Qin army and remained inside his fortifications. Qin could not break through and the armies were locked in stalemate for three years. The Zhao king decided that Lian Po was not aggressive enough and sent out Zhao Kuo who promised a decisive battle. At the same time Qin secretly replaced Wang He with the notoriously violent Bai Qi. When Zhao Kuo left his fortifications, Bai Qi used a Cannae maneuver, falling back in the center and surrounding the Zhao army from the sides. After being surrounded for 46 days, the starving Zhao troops surrendered in September 260 BC. It is said that Bai Qi had all the prisoners killed and that Zhao lost 400,000 men. - - -Qin was too exhausted to follow up its victory. Some time later it sent an army to besiege the Zhao capital but the army was destroyed when it was attacked from the rear. Zhao survived, but there was no longer a state that could resist Qin on its own. The other states could have survived if they remained united against Qin, but they did not. - - -In 257 BC, Qin army failed to besiege Handan and was defeated by the allied force of Zhao, Wei and Chu during the Battle of Handan. - - -End of Zhou dynasty (256–249 BC) - -The forces of King Zhao of Qin defeated King Nan of Zhou and conquered West Zhou in 256 BC, claiming the Nine Cauldrons and thereby symbolically becoming The Son of Heaven. - - -King Zhao's exceptionally long reign ended in 251 BC. His son King Xiaowen, already an old man, died just three days after his coronation and was succeeded by his son King Zhuangxiang of Qin. The new Qin king proceeded to conquer East Zhou, seven years after the fall of West Zhou. Thus the 800-year Zhou dynasty, nominally China's longest-ruling regime, finally came to an end. - - -Sima Qian contradicts himself regarding the ultimate fate of the East Zhou court. Chapter 4 (The Annals of Zhou) concludes with the sentence "thus the sacrifices of Zhou ended", but in the following chapter 5 (The Annals of Qin) we learn that "Qin did not prohibit their sacrifices; the Lord of Zhou was allotted a patch of land in Yangren where he could continue his ancestral sacrifices". - - -Qin unites China (247–221 BC) - -King Zhuangxiang of Qin ruled for only three years. He was succeeded by his son Zheng, who unlike the two elderly kings that preceded him was only 13 years old at his coronation. As an adult, Zheng became a brilliant commander who, in the span of just nine years, unified China. - - -Conquest of Han - -In 230 BC, Qin conquered Han. Han, the weakest of the Seven Warring States, was adjacent to the much stronger Qin, and had suffered continuous assaults by Qin in earlier years of the Warring States period. This went on until Emperor Qin Shi Huang sent general Wang Jian to attack Zhao. King An of Han, frightened by the thought that Han would be the next target of the Qin state, immediately sent diplomats to surrender the entire kingdom without a fight, saving the Han populace from the terrible potential consequences of an unsuccessful resistance. - - -Conquest of Wei - -In 225 BC, Qin conquered Wei. The Qin army led a direct invasion into Wei by besieging its capital Daliang but soon realized that the city walls were too tough to break into. They devised a new strategy in which they utilized the power of a local river that was linked to the Yellow River. The river was used to flood the city's walls, causing massive devastation to the city. Upon realizing the situation, King Jia of Wei hurriedly came out of the capital and surrendered it to the Qin army in order to avoid further bloodshed of his people. - - -Conquest of Chu - -In 223 BC, Qin conquered Chu. -The first invasion was however an utter disaster when 200,000 Qin troops, led by the general, Li Xin, were defeated by 500,000 Chu troops in the unfamiliar territory of Huaiyang, modern-day northern Jiangsu and Anhui provinces. Xiang Yan, the Chu commander, had lured Qin by allowing a few initial victories, but then counterattacked and burnt two large Qin camps. - - -In 222 BC, Wang Jian was recalled to lead a second military invasion with 600,000 men against the Chu state. High in morale after their victory in the previous year, the Chu forces were content to sit back and defend against what they expected to be a siege of Chu. However, Wang Jian decided to weaken Chu's resolve and tricked the Chu army by appearing to be idle in his fortifications whilst secretly training his troops to fight in Chu territory. After a year, the Chu defenders decided to disband due to apparent lack of action from the Qin. Wang Jian invaded at that point, with full force, and overran Huaiyang and the remaining Chu forces. Chu lost the initiative and could only sustain local guerrilla-style resistance until it too was fully conquered with the destruction of Shouchun and the death of its last leader, Lord Changping, in 223 BC. At their peak, the combined armies of Chu and Qin are estimated to have ranged from hundreds of thousands to a million soldiers, more than those involved in the campaign of Changping between Qin and Zhao 35 years earlier. - - -Conquest of Zhao and Yan - -In 222 BC, Qin conquered Zhao and Yan. -After the conquest of Zhao, the Qin army turned its attention towards Yan. Realizing the danger and gravity of this situation, Crown Prince Dan of Yan had sent Jing Ke to assassinate King Zheng of Qin, but this failure only helped to fuel the rage and determination of the Qin king, and he increased the number of troops to conquer the Yan state. - - -Conquest of Qi - -In 221 BC, Qin conquered Qi, the final unconquered state. It had not previously contributed or helped other states when Qin was conquering them. As soon as Qin's intention to invade it became clear, Qi swiftly surrendered all its cities, completing the unification of China and ushering in the Qin dynasty. The last Qi king lived out his days in exile in Gong and was not given a posthumous name, therefore he is known to posterity by his personal name Jian. - - -Aftermath - -The Qin king Ying Zheng declared himself as Qin Shi Huangdi, "The first Sovereign Emperor of Qin". - - -In the rule of the Qin state, the union was based solely on military power. The feudal holdings were abolished, and noble families were forced to live in the capital city Xianyang, in order to be supervised. A national road (as well as greater use of canals) allowed for faster and easier deployment and supply of the army. The peasants were given a wider range of land rights, although they were subject to taxation, creating a large amount of revenue to the state. - - -Military theory and practice - -Increasing scale of warfare - -The chariot remained a major factor in Chinese warfare long after it went out of fashion in the Middle East. Near the beginning of the Warring States period there is a shift from chariots to massed infantry, possibly associated with the invention of the crossbow. This had two major effects. First it led the dukes to weaken their chariot-riding nobility so they could get direct access to the peasantry who could be drafted as infantry. This change was associated with the shift from aristocratic to bureaucratic government. Second, it led to a massive increase in the scale of warfare. When the Zhou overthrew the Shang at the Battle of Muye they used 45,000 troops and 300 chariots. For the Warring States period the following figures for the military strengths of various states are reported: - - -For major battles, the following figures are reported: - - -Many scholars think these numbers are exaggerated (records are inadequate, they are much larger than those from similar societies, soldiers were paid by the number of enemies they killed and the Han dynasty had an interest in exaggerating the bloodiness of the age before China was unified). Regardless of exaggeration, it seems clear that warfare had become excessive during this period. The bloodshed and misery of the Warring States period goes a long way in explaining China's traditional and current preference for a united throne. - - -Military developments - -The Warring States period saw the introduction of many innovations to the art of warfare in China, such as the use of iron and of cavalry. - - -Warfare in the Warring States period evolved considerably from the Spring and Autumn period, as most armies made use of infantry and cavalry in battles, and the use of chariots became less widespread. The use of massed infantry made warfare bloodier and reduced the importance of the aristocracy, which in turn made the kings more despotic. From this period onward, as the various states competed with each other by mobilizing their armies to war, nobles in China belonged to the literate class, rather than to the warrior class as had previously been the case. - - -The various states fielded massive armies of infantry, cavalry, and chariots. Complex logistical systems maintained by efficient government bureaucracies were needed to supply, train, and control such large forces. The size of the armies ranged from tens of thousands to several hundred thousand men. -Iron weapons became more widespread and began to replace bronze. Most armour and weapons of this period were made from iron. - - -The first official native Chinese cavalry unit was formed in 307 BC during the military reforms of King Wuling of Zhao, who advocated 'nomadic dress and horse archery'. But the war chariot still retained its prestige and importance, despite the tactical superiority of cavalry. - - -The crossbow was the preferred long-range weapon of this period, due to several reasons. The crossbow could be mass-produced easily, and mass training of crossbowmen was possible. These qualities made it a powerful weapon against the enemy. - - -Infantrymen deployed a variety of weapons, but the most popular was the dagger-axe. The dagger-axe came in various lengths, from 9 to 18 feet; the weapon consisted of a thrusting spear with a slashing blade appended to it. Dagger-axes were an extremely popular weapon in various kingdoms, especially for the Qin, who produced 18-foot-long pike-like weapons. - - -The Qiang battle spear was named as the king 'wang' of all ancient weapons. It had the biggest impact on the battlefield and was quite difficult to master. The second important weapon of that era was the double-edged battle sword Jian. The fighting methods of using the Qiang spear and Jian sword were very different from what we see in movies or re-enactment shows today. Professional warriors of that era used the military concepts of "Master" Sun Tzu and created several successful "Ge Dou" martial schools. - - -Military thought - -The Warring States was a great period for military strategy; of the Seven Military Classics of China, four were written during this period: - - -Culture and society - -The Warring States period was an era of warfare in ancient China, as well as bureaucratic and military reforms and consolidation; the major states, ruling over large territories, quickly sought to consolidate their powers, leading to the final erosion of the Zhou court's prestige. As a sign of this shift, the rulers of all the major states (except for Chu, which had claimed kingly title much earlier) abandoned their former feudal titles for the title of 王, or King, claiming equality with the rulers of the Zhou. - - -At the same time, the constant conflict and need for innovative social and political models led to the development of many philosophical doctrines, later known as the Hundred Schools of Thought. The most notable schools of thought include Mohism (expounded by Mozi), Confucianism (represented by Mencius and Xunzi), Legalism (represented by Shang Yang, Shen Buhai, Shen Dao and Han Fei) and Taoism (represented by Zhuangzi and Lao Tzu). - - -The many states that were competing between each other attempted to display their power not only militarily but in their courts and in state philosophy. Many differing rulers adopted the differing philosophies to their own advantage or that of their kingdom. - - -Mencius attempted to instate Confucianism as a state philosophy, proposing that through the governing of moral principles like benevolence and righteousness, the state would win popular support from one state and those neighboring, eliminating the need of a war altogether. Mencius had attempted to convince King Hui of Liang, although was unsuccessful since the king saw no advantage in the period of wars. - - -Mohism was developed by Mozi (468–376 BC) and it provided a unified moral and political philosophy based on impartiality and benevolence. Mohists had the belief that people change depending on environments around. The same was applied to rulers, which is why one must be cautious of foreign influences. Mozi was very much against warfare, although he was a great tactician in defense. He defended the small state of Song from many attempts of the Chu state. - - -Taoism was advocated by Laozi, and believed that human nature was good and can achieve perfection by returning to its original state. It believed that like a baby, humans are simple and innocent although with development of civilizations it lost its innocence only to be replaced by fraud and greed. Contrarily to other schools, it did not want to gain influence in the offices of states and Laozi even refused to be the minister of the state of Chu. - - -Legalism created by Shang Yang in 338 BC, rejected all notions of religion and practices, and believed a nation should be governed by strict law. Not only were severe punishments applied, but they would be grouped with the families and made mutually responsible for criminal act. It proposed radical reforms, and established a society based on solid ranks. Peasants were encouraged to practice agriculture as occupation, and military performance was rewarded. Laws were also applied to all ranks with no exception; even the king was not above punishment. The philosophy was adapted by the Qin state and it created it into an organized, centralized state with a bureaucracy chosen on the basis of merit. -This period is most famous for the establishment of complex bureaucracies and centralized governments, as well as a clear legal system. The developments in political and military organization were the basis of the power of the Qin state, which conquered the other states and unified them under the Qin dynasty in 221 BC. - - -Nobles, bureaucrats and reformers - -The phenomenon of intensive warfare, based on mass formations of infantry rather than the traditional chariots, was one major trend which led to the creation of strong central bureaucracies in each of the major states. At the same time, the process of secondary feudalism which permeated the Spring and Autumn period, and led to such events as the partition of Jin and the usurpation of Qi by the Tian clan, was eventually reversed by the same process of bureaucratisation. - - -Under the demands of warfare, the states adopted bureaucratic reforms in the Warring States period. Wei adopted these in 445 BC, Zhao in 403 BC, Chu in 390 BC, Han in 355 BC, Qi in 357 BC and Qin in 350 BC. Power was centralised by curbing the landed aristocrats and sinecures and creating a new hierarchy based on meritorious service to the state, which were drawn from the lower rungs of society. Systematic auditing and reporting systems, and fixed salaries for officials were created. - - -The reforms of Shang Yang in Qin, and of Wu Qi in Chu, both centred on increased centralisation, the suppression of the nobility, and a vastly increased scope of government based on Legalist ideals, which were necessary to mobilise the large armies of the period. - - -Sophisticated arithmetic - -A bundle of 21 bamboo slips from the Tsinghua collection dated to 305 BC are the world's earliest example of a two digit decimal multiplication table, indicating that sophisticated commercial arithmetic was already established during this period. - - -Rod numerals were used to represent both negative and positive integers, and rational numbers, a true positional number system, with a blank for zero dating back to the Warring States period. - - -The nine linked-rings puzzle, an advanced puzzle device which requires mathematical analysis to solve, was invented during the period. - - -Literature - -An important literary achievement of the Warring States period is the Zuo Commentary on the Spring and Autumn Annals, which summarizes the preceding Spring and Autumn period. The less famous work Guoyu is thought to be by the same author. - - -Many sayings of Spring and Autumn philosophers, which had previously been circulated orally, were put into writing in the Warring States. These include the Analects and The Art of War. - - -Economic developments - -The Warring States period saw the proliferation of iron working in China, replacing bronze as the dominant type of metal used in warfare. Areas such as Shu (present-day Sichuan) and Yue (present-day Zhejiang) were also brought into the Chinese cultural sphere during this time. Trade also became important, and some merchants had considerable power in politics, the most prominent of which was Lü Buwei, who rose to become Chancellor of Qin and was a key supporter of the eventual Qin Shihuang. - - -At the same time, the increased resources of consolidated, bureaucratic states, coupled with the logistical needs of mass levies and large-scale warfare, led to the proliferation of economic projects such as large-scale waterworks. Major examples of such waterworks include the Dujiangyan Irrigation System, which controlled the Min River in Sichuan and turned the former backwater region into a major Qin logistical base, and the Zhengguo Canal which irrigated large areas of land in the Guanzhong Plain, again increasing Qin's agricultural output. - - -The Guanzi is considered one of the most foundational texts of the developing political economy in the Warring States period. It addresses principles of price regulation in the context of effectively dealing with commodities that are "light" (connoting a commodity which is unimportant, non-essential, or inexpensive) or "heavy" (a commodity which is important, essential, or expensive) and how whether a commodity is "light" or "heavy" is understood in relation to other commodities. - - -In summary:""" - # fmt: on + generation_args = { + "max_new_tokens": 1000, + "temperature": 0.0, + "do_sample": False, + } - input_ids = tokenizer(LONG_TEXT, return_tensors="pt").input_ids.to(torch_device) - generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] + predictions = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args) + predictions = predictions[:, inputs['input_ids'].shape[1]:] - torch.testing.assert_close(generated_ids.cpu(), torch.tensor([[ 576, 4570, 71818, 374, 6509, 825, 315, 279, 1429, 88228, 21984, 315, 279, 11220, 4948, 8584, 304, 279, 467, 19859, 4180, 4168, 13, 1084, 14230, 16170, 315, 3349, 19256, 304, 279, 2266, 315, 13444, 14550, 448, 50867, 429, 525, 330, 4145, 1, 476, 330, 88845, 1, 323, 1246, 3425, 264]], dtype=torch.long)) # fmt: skip self.assertEqual( - tokenizer.decode(generated_ids[0]), - """ The Guanzi is considered one of the most foundational texts of the developing political economy in the Warring States period. It addresses principles of price regulation in the context of effectively dealing with commodities that are "light" or "heavy" and how whether a""", + processor.batch_decode(predictions, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0], + "The black SUV is in motion, as indicated by the position of the wheels and the blurred background." ) - model.config._attn_implementation = "eager" - new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] - with self.subTest("Eager matches sdpa"): - torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) - - model.config._attn_implementation = "flex_attention" - new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] - with self.subTest("Eager matches Flex attention"): - torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) - - model.config._attn_implementation = "flash_attention_2" - new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :] - with self.subTest("Eager matches flash attention"): - torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4) From 502fad6276c6c45048d1463408147de038c3617e Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 3 Mar 2025 19:01:29 +0000 Subject: [PATCH 20/48] Add requests + make style --- tests/models/phi3_v/test_modeling_phi3_v.py | 30 +++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py index 411ffab3df0c..5be563326247 100644 --- a/tests/models/phi3_v/test_modeling_phi3_v.py +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -14,23 +14,19 @@ # limitations under the License. """Testing suite for the PyTorch Phi3V model.""" -import gc import unittest import pytest -from packaging import version +import requests -from transformers import AutoTokenizer, Phi3VConfig, is_torch_available, is_vision_available, set_seed -from transformers.generation.configuration_utils import GenerationConfig +from transformers import Phi3VConfig, is_torch_available, is_vision_available from transformers.testing_utils import ( - backend_empty_cache, - require_bitsandbytes, require_flash_attn, require_torch, require_torch_gpu, - require_torch_sdpa, + require_vision, slow, - torch_device, require_vision, + torch_device, ) from ...generation.test_utils import GenerationTesterMixin @@ -447,21 +443,15 @@ def get_images(): class Phi3VIntegrationTest(unittest.TestCase): def test_casual_generate(self): model = Phi3VForCausalLM.from_pretrained( - "microsoft/Phi-3.5-vision-instruct", - torch_dtype="auto", - _attn_implementation='flash_attention_2').to(torch_device) - processor = Phi3VProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", - num_crops=4) + "microsoft/Phi-3.5-vision-instruct", torch_dtype="auto", _attn_implementation="flash_attention_2" + ).to(torch_device) + processor = Phi3VProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", num_crops=4) images = get_images() messages = [ {"role": "user", "content": "<|image_1|>\nWhat in this image is in motion?"}, ] - prompt = processor.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - ) + prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = processor(prompt, images, return_tensors="pt").to(torch_device) generation_args = { @@ -471,9 +461,9 @@ def test_casual_generate(self): } predictions = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args) - predictions = predictions[:, inputs['input_ids'].shape[1]:] + predictions = predictions[:, inputs["input_ids"].shape[1] :] self.assertEqual( processor.batch_decode(predictions, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0], - "The black SUV is in motion, as indicated by the position of the wheels and the blurred background." + "The black SUV is in motion, as indicated by the position of the wheels and the blurred background.", ) From d39d36e8b4f8c9294d062822f1d7ddad5a149e8a Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 3 Mar 2025 19:07:32 +0000 Subject: [PATCH 21/48] Add what I think is the docstring for num_crops --- src/transformers/models/phi3_v/image_processing_phi3_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index d37761a8daa3..6a38203e524b 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -138,7 +138,7 @@ class Phi3VImageProcessor(BaseImageProcessor): for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512) Args: - num_crops (`int`, *optional*, defaults to 1): + num_crops (`int`, *optional*, defaults to 1): Number of different image crops to use. image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. From ba2ea311a89514ab55907112358b1f2a889f5b85 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 3 Mar 2025 16:29:31 -0700 Subject: [PATCH 22/48] changed _attn_implementation to eager --- tests/models/phi3_v/test_modeling_phi3_v.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py index 5be563326247..39389e0b6901 100644 --- a/tests/models/phi3_v/test_modeling_phi3_v.py +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -443,7 +443,7 @@ def get_images(): class Phi3VIntegrationTest(unittest.TestCase): def test_casual_generate(self): model = Phi3VForCausalLM.from_pretrained( - "microsoft/Phi-3.5-vision-instruct", torch_dtype="auto", _attn_implementation="flash_attention_2" + "microsoft/Phi-3.5-vision-instruct", torch_dtype="auto", _attn_implementation="eager" ).to(torch_device) processor = Phi3VProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", num_crops=4) images = get_images() @@ -455,8 +455,7 @@ def test_casual_generate(self): inputs = processor(prompt, images, return_tensors="pt").to(torch_device) generation_args = { - "max_new_tokens": 1000, - "temperature": 0.0, + "max_new_tokens": 50, "do_sample": False, } From 087505ffcad72c1afa8e5011e06b1516cf6e47c1 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 7 Mar 2025 16:28:38 +0000 Subject: [PATCH 23/48] Push a half-hearted image processor test --- .../phi3_v/test_image_processing_phi3_v.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 tests/models/phi3_v/test_image_processing_phi3_v.py diff --git a/tests/models/phi3_v/test_image_processing_phi3_v.py b/tests/models/phi3_v/test_image_processing_phi3_v.py new file mode 100644 index 000000000000..2b109c058879 --- /dev/null +++ b/tests/models/phi3_v/test_image_processing_phi3_v.py @@ -0,0 +1,188 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np + +from transformers.image_utils import PILImageResampling +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available + +from ...test_image_processing_common import ImageProcessingTestMixin + + +if is_vision_available(): + from PIL import Image + + from transformers import Phi3VImageProcessor + + +if is_torch_available(): + import torch + + +class Phi3VImageProcessingTester: + def __init__( + self, + parent, + batch_size=7, + num_channels=3, + num_images=1, + image_size=18, + min_resolution=30, + max_resolution=40, + do_resize=True, + size=None, + max_image_size=None, + do_rescale=True, + rescale_factor=1 / 255, + do_normalize=True, + image_mean=[0.5, 0.5, 0.5], + image_std=[0.5, 0.5, 0.5], + do_convert_rgb=True, + do_pad=True, + do_image_splitting=True, + ): + self.size = size if size is not None else {"longest_edge": max_resolution} + self.parent = parent + self.batch_size = batch_size + self.num_channels = num_channels + self.num_images = num_images + self.image_size = image_size + self.min_resolution = min_resolution + self.max_resolution = max_resolution + self.do_resize = do_resize + self.do_image_splitting = do_image_splitting + self.max_image_size = max_image_size if max_image_size is not None else {"longest_edge": 20} + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean + self.image_std = image_std + self.do_convert_rgb = do_convert_rgb + self.do_pad = do_pad + + def prepare_image_processor_dict(self): + return { + "do_convert_rgb": self.do_convert_rgb, + "do_resize": self.do_resize, + "size": self.size, + "max_image_size": self.max_image_size, + "do_rescale": self.do_rescale, + "rescale_factor": self.rescale_factor, + "do_normalize": self.do_normalize, + "image_mean": self.image_mean, + "image_std": self.image_std, + "do_pad": self.do_pad, + "do_image_splitting": self.do_image_splitting, + } + + def get_expected_values(self, image_inputs, batched=False): + """ + This function computes the expected height and width when providing images to Phi3VImageProcessor, + assuming do_resize is set to True. The expected size in that case the max image size. + """ + return self.max_image_size["longest_edge"], self.max_image_size["longest_edge"] + + def expected_output_image_shape(self, images): + height, width = self.get_expected_values(images, batched=True) + effective_nb_images = ( + self.num_images * 5 if self.do_image_splitting else 1 + ) # 5 is a squared image divided into 4 + global image resized + return effective_nb_images, self.num_channels, height, width + + def prepare_image_inputs( + self, + batch_size=None, + min_resolution=None, + max_resolution=None, + num_channels=None, + num_images=None, + size_divisor=None, + equal_resolution=False, + numpify=False, + torchify=False, + ): + """This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True, + or a list of PyTorch tensors if one specifies torchify=True. + + One can specify whether the images are of the same resolution or not. + """ + assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time" + + batch_size = batch_size if batch_size is not None else self.batch_size + min_resolution = min_resolution if min_resolution is not None else self.min_resolution + max_resolution = max_resolution if max_resolution is not None else self.max_resolution + num_channels = num_channels if num_channels is not None else self.num_channels + num_images = num_images if num_images is not None else self.num_images + + images_list = [] + for i in range(batch_size): + images = [] + for j in range(num_images): + if equal_resolution: + width = height = max_resolution + else: + # To avoid getting image width/height 0 + if size_divisor is not None: + # If `size_divisor` is defined, the image needs to have width/size >= `size_divisor` + min_resolution = max(size_divisor, min_resolution) + width, height = np.random.choice(np.arange(min_resolution, max_resolution), 2) + images.append(np.random.randint(255, size=(num_channels, width, height), dtype=np.uint8)) + images_list.append(images) + + if not numpify and not torchify: + # PIL expects the channel dimension as last dimension + images_list = [[Image.fromarray(np.moveaxis(image, 0, -1)) for image in images] for images in images_list] + + if torchify: + images_list = [[torch.from_numpy(image) for image in images] for images in images_list] + + if numpify: + # Numpy images are typically in channels last format + images_list = [[image.transpose(1, 2, 0) for image in images] for images in images_list] + + return images_list + + +@require_torch +@require_vision +class Phi3VImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): + image_processing_class = Phi3VImageProcessor if is_vision_available() else None + + def setUp(self): + super().setUp() + self.image_processor_tester = Phi3VImageProcessingTester(self) + + @property + def image_processor_dict(self): + return self.image_processor_tester.prepare_image_processor_dict() + + def test_image_processor_properties(self): + image_processing = self.image_processing_class(**self.image_processor_dict) + self.assertTrue(hasattr(image_processing, "do_convert_rgb")) + self.assertTrue(hasattr(image_processing, "do_resize")) + self.assertTrue(hasattr(image_processing, "size")) + self.assertTrue(hasattr(image_processing, "do_image_splitting")) + self.assertTrue(hasattr(image_processing, "max_image_size")) + self.assertTrue(hasattr(image_processing, "do_rescale")) + self.assertTrue(hasattr(image_processing, "rescale_factor")) + self.assertTrue(hasattr(image_processing, "do_normalize")) + self.assertTrue(hasattr(image_processing, "image_mean")) + self.assertTrue(hasattr(image_processing, "image_std")) + self.assertTrue(hasattr(image_processing, "do_pad")) + self.assertTrue(hasattr(image_processing, "do_image_splitting")) \ No newline at end of file From ea43ebd992216f7be2f052029f3cdf728e972cdd Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sat, 8 Mar 2025 03:52:32 -0700 Subject: [PATCH 24/48] various changes attempting to make ground on the image processor tests --- .../models/phi3_v/image_processing_phi3_v.py | 25 ++++++++++++++----- .../phi3_v/test_image_processing_phi3_v.py | 6 ++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index 6a38203e524b..731f0a49a3a6 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -72,6 +72,14 @@ def calc_padded_size(width, height, padding_unit=336): def HD_transform(img, hd_num=16): + #breakpoint() + is_tensor = isinstance(img, torch.Tensor) + is_numpy = isinstance(img, np.ndarray) + + if is_numpy: + img = Image.fromarray(img) + elif is_tensor: + img = torchvision.transforms.functional.to_pil_image(img) width, height = img.size trans = False if width < height: @@ -91,10 +99,14 @@ def HD_transform(img, hd_num=16): [new_h, new_w], ) img = padding_336(img) - width, height = img.size if trans: img = img.transpose(Image.TRANSPOSE) + if is_tensor: + img = torchvision.transforms.functional.to_tensor(img) + elif is_numpy: + img = np.asarray(img) + return img @@ -155,6 +167,7 @@ class Phi3VImageProcessor(BaseImageProcessor): def __init__( self, num_crops: int = 1, + num_img_tokens: int = 144, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, do_convert_rgb: bool = True, @@ -162,6 +175,7 @@ def __init__( ) -> None: super().__init__(**kwargs) self.num_crops = num_crops + self.num_img_tokens = num_img_tokens self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD self.do_convert_rgb = do_convert_rgb @@ -206,6 +220,7 @@ def preprocess( image_std: Optional[Union[float, List[float]]] = None, do_convert_rgb: bool = None, return_tensors: Optional[Union[str, TensorType]] = None, + **kwargs, ): """ Args: @@ -231,7 +246,8 @@ def preprocess( image_std = image_std if image_std is not None else self.image_std do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - images = make_list_of_images(images) + if not isinstance(images, List): + images = make_list_of_images(images) if not valid_images(images): raise ValueError( @@ -239,9 +255,6 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - if do_convert_rgb: - images = [convert_to_rgb(image) for image in images] - image_sizes = [] img_processor = torchvision.transforms.Compose( [torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(image_mean, image_std)] @@ -250,7 +263,7 @@ def preprocess( # PIL images # HD_transform pad images to size of multiiply of 336, 336 # convert to RGB first - images = [image.convert("RGB") for image in images] + images = [convert_to_rgb(image) for image in images] elems = [HD_transform(im, hd_num=self.num_crops) for im in images] # tensor transform and normalize hd_images = [img_processor(im) for im in elems] diff --git a/tests/models/phi3_v/test_image_processing_phi3_v.py b/tests/models/phi3_v/test_image_processing_phi3_v.py index 2b109c058879..6829fd2b195e 100644 --- a/tests/models/phi3_v/test_image_processing_phi3_v.py +++ b/tests/models/phi3_v/test_image_processing_phi3_v.py @@ -147,14 +147,14 @@ def prepare_image_inputs( if not numpify and not torchify: # PIL expects the channel dimension as last dimension - images_list = [[Image.fromarray(np.moveaxis(image, 0, -1)) for image in images] for images in images_list] + images_list = [Image.fromarray(np.moveaxis(image, 0, -1)) for image in images] if torchify: - images_list = [[torch.from_numpy(image) for image in images] for images in images_list] + images_list = [torch.from_numpy(image) for image in images] if numpify: # Numpy images are typically in channels last format - images_list = [[image.transpose(1, 2, 0) for image in images] for images in images_list] + images_list = [image.transpose(1, 2, 0) for image in images] return images_list From e49c24def79c0e7cd96c93a9be6cf143f234b1c8 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sat, 8 Mar 2025 20:56:18 -0700 Subject: [PATCH 25/48] All tests passing except test_call_pytorch, still working on that! --- .../models/phi3_v/image_processing_phi3_v.py | 1 - .../phi3_v/test_image_processing_phi3_v.py | 48 ++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index 731f0a49a3a6..6e232b1cd34b 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -72,7 +72,6 @@ def calc_padded_size(width, height, padding_unit=336): def HD_transform(img, hd_num=16): - #breakpoint() is_tensor = isinstance(img, torch.Tensor) is_numpy = isinstance(img, np.ndarray) diff --git a/tests/models/phi3_v/test_image_processing_phi3_v.py b/tests/models/phi3_v/test_image_processing_phi3_v.py index 6829fd2b195e..ffab8f464f44 100644 --- a/tests/models/phi3_v/test_image_processing_phi3_v.py +++ b/tests/models/phi3_v/test_image_processing_phi3_v.py @@ -51,11 +51,12 @@ def __init__( do_rescale=True, rescale_factor=1 / 255, do_normalize=True, - image_mean=[0.5, 0.5, 0.5], - image_std=[0.5, 0.5, 0.5], + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], do_convert_rgb=True, do_pad=True, - do_image_splitting=True, + #do_image_splitting=True, + num_crops=1 ): self.size = size if size is not None else {"longest_edge": max_resolution} self.parent = parent @@ -66,8 +67,8 @@ def __init__( self.min_resolution = min_resolution self.max_resolution = max_resolution self.do_resize = do_resize - self.do_image_splitting = do_image_splitting - self.max_image_size = max_image_size if max_image_size is not None else {"longest_edge": 20} + #self.do_image_splitting = do_image_splitting + self.max_image_size = max_image_size if max_image_size is not None else {"longest_edge": 336} self.do_rescale = do_rescale self.rescale_factor = rescale_factor self.do_normalize = do_normalize @@ -75,6 +76,7 @@ def __init__( self.image_std = image_std self.do_convert_rgb = do_convert_rgb self.do_pad = do_pad + self.num_crops = num_crops def prepare_image_processor_dict(self): return { @@ -88,7 +90,8 @@ def prepare_image_processor_dict(self): "image_mean": self.image_mean, "image_std": self.image_std, "do_pad": self.do_pad, - "do_image_splitting": self.do_image_splitting, + #"do_image_splitting": self.do_image_splitting, + "num_crops": self.num_crops } def get_expected_values(self, image_inputs, batched=False): @@ -101,7 +104,8 @@ def get_expected_values(self, image_inputs, batched=False): def expected_output_image_shape(self, images): height, width = self.get_expected_values(images, batched=True) effective_nb_images = ( - self.num_images * 5 if self.do_image_splitting else 1 + #self.num_images * 5 if self.do_image_splitting else 1 + (self.num_crops + 1) * self.num_images ) # 5 is a squared image divided into 4 + global image resized return effective_nb_images, self.num_channels, height, width @@ -131,8 +135,8 @@ def prepare_image_inputs( num_images = num_images if num_images is not None else self.num_images images_list = [] + images = [] for i in range(batch_size): - images = [] for j in range(num_images): if equal_resolution: width = height = max_resolution @@ -143,7 +147,6 @@ def prepare_image_inputs( min_resolution = max(size_divisor, min_resolution) width, height = np.random.choice(np.arange(min_resolution, max_resolution), 2) images.append(np.random.randint(255, size=(num_channels, width, height), dtype=np.uint8)) - images_list.append(images) if not numpify and not torchify: # PIL expects the channel dimension as last dimension @@ -177,7 +180,6 @@ def test_image_processor_properties(self): self.assertTrue(hasattr(image_processing, "do_convert_rgb")) self.assertTrue(hasattr(image_processing, "do_resize")) self.assertTrue(hasattr(image_processing, "size")) - self.assertTrue(hasattr(image_processing, "do_image_splitting")) self.assertTrue(hasattr(image_processing, "max_image_size")) self.assertTrue(hasattr(image_processing, "do_rescale")) self.assertTrue(hasattr(image_processing, "rescale_factor")) @@ -185,4 +187,28 @@ def test_image_processor_properties(self): self.assertTrue(hasattr(image_processing, "image_mean")) self.assertTrue(hasattr(image_processing, "image_std")) self.assertTrue(hasattr(image_processing, "do_pad")) - self.assertTrue(hasattr(image_processing, "do_image_splitting")) \ No newline at end of file + self.assertTrue(hasattr(image_processing, "num_crops")) + + def test_call_numpy_4_channels(self): + # Phi3V always processes images as RGB, so it always returns images with 3 channels + for image_processing_class in self.image_processor_list: + # Initialize image_processing + image_processor_dict = self.image_processor_dict + image_processing = self.image_processing_class(**image_processor_dict) + # create random numpy tensors + image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, numpify=True) + + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + # Test not batched input + encoded_images = image_processing(image_inputs[0], return_tensors="pt").pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape([image_inputs[0]]) + self.assertEqual(tuple(encoded_images.shape), (1, *expected_output_image_shape)) + + # Test batched + encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values + expected_output_image_shape = self.image_processor_tester.expected_output_image_shape(image_inputs) + self.assertEqual( + tuple(encoded_images.shape), (self.image_processor_tester.batch_size, *expected_output_image_shape) + ) From f35c6db05708a6dff986c978ba7a22a820180ea8 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sat, 8 Mar 2025 21:04:31 -0700 Subject: [PATCH 26/48] Nevermind, all green --- src/transformers/models/phi3_v/image_processing_phi3_v.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index 6e232b1cd34b..1da2f1d6bf5a 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -101,11 +101,6 @@ def HD_transform(img, hd_num=16): if trans: img = img.transpose(Image.TRANSPOSE) - if is_tensor: - img = torchvision.transforms.functional.to_tensor(img) - elif is_numpy: - img = np.asarray(img) - return img From c27d546e3c2555844f09b72671b219ef25af2152 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 9 Mar 2025 00:58:32 -0700 Subject: [PATCH 27/48] Cleaned up some frass and remembered to run make fix-copies --- src/transformers/models/phi3_v/image_processing_phi3_v.py | 5 ++--- tests/models/phi3_v/test_image_processing_phi3_v.py | 7 +------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index 1da2f1d6bf5a..b37330eb12f3 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -145,6 +145,7 @@ class Phi3VImageProcessor(BaseImageProcessor): Args: num_crops (`int`, *optional*, defaults to 1): Number of different image crops to use. + num_img_tokens (`int`, *optional*, defaults to 144): image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. @@ -249,13 +250,12 @@ def preprocess( "torch.Tensor, tf.Tensor or jax.ndarray." ) - image_sizes = [] img_processor = torchvision.transforms.Compose( [torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(image_mean, image_std)] ) # PIL images - # HD_transform pad images to size of multiiply of 336, 336 + # HD_transform pad images to size of multiply of 336, 336 # convert to RGB first images = [convert_to_rgb(image) for image in images] elems = [HD_transform(im, hd_num=self.num_crops) for im in images] @@ -291,7 +291,6 @@ def preprocess( # pad to max_num_crops image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops + 1) for im in hd_images_reshape] image_transformed = torch.stack(image_transformed, dim=0) - image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes] padded_images = image_transformed image_sizes = shapes diff --git a/tests/models/phi3_v/test_image_processing_phi3_v.py b/tests/models/phi3_v/test_image_processing_phi3_v.py index ffab8f464f44..396db35651b0 100644 --- a/tests/models/phi3_v/test_image_processing_phi3_v.py +++ b/tests/models/phi3_v/test_image_processing_phi3_v.py @@ -18,7 +18,6 @@ import numpy as np -from transformers.image_utils import PILImageResampling from transformers.testing_utils import require_torch, require_vision from transformers.utils import is_torch_available, is_vision_available @@ -55,7 +54,6 @@ def __init__( image_std=[0.26862954, 0.26130258, 0.27577711], do_convert_rgb=True, do_pad=True, - #do_image_splitting=True, num_crops=1 ): self.size = size if size is not None else {"longest_edge": max_resolution} @@ -67,7 +65,6 @@ def __init__( self.min_resolution = min_resolution self.max_resolution = max_resolution self.do_resize = do_resize - #self.do_image_splitting = do_image_splitting self.max_image_size = max_image_size if max_image_size is not None else {"longest_edge": 336} self.do_rescale = do_rescale self.rescale_factor = rescale_factor @@ -90,7 +87,6 @@ def prepare_image_processor_dict(self): "image_mean": self.image_mean, "image_std": self.image_std, "do_pad": self.do_pad, - #"do_image_splitting": self.do_image_splitting, "num_crops": self.num_crops } @@ -104,9 +100,8 @@ def get_expected_values(self, image_inputs, batched=False): def expected_output_image_shape(self, images): height, width = self.get_expected_values(images, batched=True) effective_nb_images = ( - #self.num_images * 5 if self.do_image_splitting else 1 (self.num_crops + 1) * self.num_images - ) # 5 is a squared image divided into 4 + global image resized + ) return effective_nb_images, self.num_channels, height, width def prepare_image_inputs( From 2ebd42e6e15a67e9c41da1308375ebf50150d1bc Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Mon, 10 Mar 2025 11:30:33 -0700 Subject: [PATCH 28/48] Pushing an equally half-hearted processor test --- tests/models/phi3_v/test_processor_phi3_v.py | 603 +++++++++++++++++++ 1 file changed, 603 insertions(+) create mode 100644 tests/models/phi3_v/test_processor_phi3_v.py diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py new file mode 100644 index 000000000000..dae0a4c0896f --- /dev/null +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -0,0 +1,603 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shutil +import tempfile +import unittest +from io import BytesIO +from typing import Optional + +import numpy as np +import requests + +from transformers import Phi3VProcessor +from transformers.models.auto.processing_auto import AutoProcessor +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_vision_available + +from ...test_processing_common import ProcessorTesterMixin + + +if is_vision_available(): + from PIL import Image + + +@require_torch +@require_vision +class Phi3VProcessorTest(ProcessorTesterMixin, unittest.TestCase): + processor_class = Phi3VProcessor + videos_input_name = "pixel_values" + + @classmethod + def setUpClass(cls): + cls.tmpdirname = tempfile.mkdtemp() + processor = Phi3VProcessor.from_pretrained("microsoft/Phi-3.5-vision-instruct", num_crops=4) + processor.save_pretrained(cls.tmpdirname) + cls.image1 = Image.open( + BytesIO( + requests.get( + "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + ).content + ) + ) + cls.image2 = Image.open( + BytesIO(requests.get("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg").content) + ) + cls.image3 = Image.open( + BytesIO( + requests.get( + "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg" + ).content + ) + ) + cls.bos_token = processor.tokenizer.bos_token + cls.image_token = processor.image_token + cls.fake_image_token = processor.fake_image_token + cls.global_img_token = processor.global_image_token + + cls.bos_token_id = processor.tokenizer.convert_tokens_to_ids(cls.bos_token) + cls.image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.image_token) + cls.fake_image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.fake_image_token) + cls.global_img_tokens_id = processor.tokenizer(cls.global_img_token, add_special_tokens=False)["input_ids"] + cls.padding_token_id = processor.tokenizer.pad_token_id + cls.image_seq_len = processor.image_seq_len + + def get_tokenizer(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer + + def get_image_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor + + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) + + def prepare_processor_dict(self): + return { + "image_seq_len": self.image_seq_len, + "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %}\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", + } + + def get_split_image_expected_tokens(self, processor, image_rows, image_cols): + text_split_images = [] + for n_h in range(image_rows): + for n_w in range(image_cols): + text_split_images += ( + [self.fake_image_token_id] + + processor.tokenizer(f"", add_special_tokens=False)["input_ids"] + + [self.image_token_id] * self.image_seq_len + ) + text_split_images += processor.tokenizer("\n", add_special_tokens=False)["input_ids"] + text_split_images = text_split_images[:-1] # remove last newline + # add double newline, as it gets its own token + text_split_images += processor.tokenizer("\n\n", add_special_tokens=False)["input_ids"] + text_split_images += ( + [self.fake_image_token_id] + + self.global_img_tokens_id + + [self.image_token_id] * self.image_seq_len + + [self.fake_image_token_id] + ) + return text_split_images + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdirname) + + def test_process_interleaved_images_prompts_no_image_splitting(self): + processor_components = self.prepare_components() + processor_components["tokenizer"] = self.get_component("tokenizer", padding_side="left") + processor_components["image_processor"] = self.get_component("image_processor", do_image_splitting=False) + processor_kwargs = self.prepare_processor_dict() + + processor = self.processor_class(**processor_components, **processor_kwargs) + + # Test that a single image is processed correctly + inputs = processor(images=self.image1) + image1_expected_size = (512, 512) + self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 1, 3, *image1_expected_size)) + self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 1, *image1_expected_size)) + # fmt: on + + # Test a single sample with image and text + image_str = "" + text_str = "In this image, we see" + text = image_str + text_str + inputs = processor(text=text, images=self.image1) + + # fmt: off + tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) + expected_input_ids = [[self.fake_image_token_id] + self.global_img_tokens_id + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + tokenized_sentence["input_ids"]] + self.assertEqual(inputs["input_ids"], expected_input_ids) + self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])]) + self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 1, 3, *image1_expected_size)) + self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 1, *image1_expected_size)) + # fmt: on + + # Test that batch is correctly processed + image_str = "" + text_str_1 = "In this image, we see" + text_str_2 = "In this image, we see" + + text = [ + image_str + text_str_1, + image_str + image_str + text_str_2, + ] + images = [[self.image1], [self.image2, self.image3]] + + inputs = processor(text=text, images=images, padding=True) + + # fmt: off + tokenized_sentence_1 = processor.tokenizer(text_str_1, add_special_tokens=False) + tokenized_sentence_2 = processor.tokenizer(text_str_2, add_special_tokens=False) + image_tokens = [self.fake_image_token_id] + self.global_img_tokens_id + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + expected_input_ids_1 = image_tokens + tokenized_sentence_1["input_ids"] + expected_input_ids_2 = 2 * image_tokens + tokenized_sentence_2["input_ids"] + # Pad the first input to match the second input + pad_len = len(expected_input_ids_2) - len(expected_input_ids_1) + padded_expected_input_ids_1 = [self.padding_token_id] * pad_len + expected_input_ids_1 + + self.assertEqual( + inputs["input_ids"], [padded_expected_input_ids_1, expected_input_ids_2] + ) + self.assertEqual( + inputs["attention_mask"], + [[0] * pad_len + [1] * len(expected_input_ids_1), [1] * len(expected_input_ids_2)] + ) + self.assertEqual(np.array(inputs['pixel_values']).shape, (2, 2, 3, 512, 512)) + self.assertEqual(np.array(inputs['pixel_attention_mask']).shape, (2, 2, 512, 512)) + # fmt: on + + def test_process_interleaved_images_prompts_image_splitting(self): + processor_components = self.prepare_components() + processor_components["tokenizer"] = self.get_component("tokenizer", padding_side="left") + processor_components["image_processor"] = self.get_component("image_processor", do_image_splitting=True) + processor_kwargs = self.prepare_processor_dict() + + processor = self.processor_class(**processor_components, **processor_kwargs) + + # Test that a single image is processed correctly + inputs = processor(images=self.image1) + self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 13, 3, 512, 512)) + self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 13, 512, 512)) + # fmt: on + self.maxDiff = None + + # Test a single sample with image and text + image_str = "" + text_str = "In this image, we see" + text = image_str + text_str + inputs = processor(text=text, images=self.image1) + + # fmt: off + tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) + split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) + expected_input_ids_1 = [split_image1_tokens + tokenized_sentence["input_ids"]] + self.assertEqual(inputs["input_ids"], expected_input_ids_1) + self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids_1[0])]) + self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 13, 3, 512, 512)) + self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 13, 512, 512)) + # fmt: on + + # Test that batch is correctly processed + image_str = "" + text_str_1 = "In this image, we see" + text_str_2 = "bla, bla" + + text = [ + image_str + text_str_1, + text_str_2 + image_str + image_str, + ] + images = [[self.image1], [self.image2, self.image3]] + + inputs = processor(text=text, images=images, padding=True) + + # fmt: off + tokenized_sentence_1 = processor.tokenizer(text_str_1, add_special_tokens=False) + tokenized_sentence_2 = processor.tokenizer(text_str_2, add_special_tokens=False) + + split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) + split_image2_tokens = self.get_split_image_expected_tokens(processor, 4, 4) + split_image3_tokens = self.get_split_image_expected_tokens(processor, 3, 4) + expected_input_ids_1 = split_image1_tokens + tokenized_sentence_1["input_ids"] + expected_input_ids_2 = tokenized_sentence_2["input_ids"] + split_image2_tokens + split_image3_tokens + # Pad the first input to match the second input + pad_len = len(expected_input_ids_2) - len(expected_input_ids_1) + padded_expected_input_ids_1 = [self.padding_token_id] * pad_len + expected_input_ids_1 + + self.assertEqual( + inputs["input_ids"], [padded_expected_input_ids_1, expected_input_ids_2] + ) + self.assertEqual( + inputs["attention_mask"], + [[0] * pad_len + [1] * len(expected_input_ids_1), [1] * len(expected_input_ids_2)] + ) + self.assertEqual(np.array(inputs['pixel_values']).shape, (2, 30, 3, 512, 512)) + self.assertEqual(np.array(inputs['pixel_attention_mask']).shape, (2, 30, 512, 512)) + # fmt: on + + def test_add_special_tokens_processor(self): + processor = self.get_processor() + + image_str = "" + text_str = "In this image, we see" + text = text_str + image_str + + # fmt: off + inputs = processor(text=text, images=self.image1, add_special_tokens=False) + tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) + split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) + expected_input_ids = [tokenized_sentence["input_ids"] + split_image1_tokens] + self.assertEqual(inputs["input_ids"], expected_input_ids) + + inputs = processor(text=text, images=self.image1) + expected_input_ids = [tokenized_sentence["input_ids"] + split_image1_tokens] + self.assertEqual(inputs["input_ids"], expected_input_ids) + # fmt: on + + @unittest.skip(reason="from @molbap @zucchini-nlp, passing non-nested images is error-prone and not recommended") + def test_non_nested_images_with_batched_text(self): + processor = self.get_processor() + processor.image_processor.do_image_splitting = False + + image_str = "" + text_str_1 = "In this image, we see" + text_str_2 = "In this image, we see" + + text = [ + image_str + text_str_1, + image_str + image_str + text_str_2, + ] + images = [[self.image1], [self.image2, self.image3]] + + inputs = processor(text=text, images=images, padding=True) + + self.assertEqual(np.array(inputs["pixel_values"]).shape, (2, 2, 3, 512, 512)) + self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (2, 2, 512, 512)) + + # Copied from tests.models.idefics2.test_processor_idefics2.Idefics2ProcessorTest.test_process_interleaved_images_prompts_image_error + def test_process_interleaved_images_prompts_image_error(self): + processor = self.get_processor() + + text = [ + "This is a test sentence.", + "In this other sentence we try some good things", + ] + images = [[self.image1], [self.image2]] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [[self.image1], []] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + + text = [ + "This is a test sentence.", + "In this other sentence we try some good things", + ] + images = [[self.image1], [self.image2, self.image3]] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [[], [self.image2]] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [self.image1, self.image2, self.image3] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [self.image1] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + + text = [ + "This is a test sentence.", + "In this other sentence we try some good things", + ] + images = [[self.image1], []] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [[], [self.image2]] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [self.image1, self.image2] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + images = [self.image1] + with self.assertRaises(ValueError): + processor(text=text, images=images, padding=True) + + def test_apply_chat_template(self): + # Message contains content which a mix of lists with images and image urls and string + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What do these images show?"}, + {"type": "image"}, + {"type": "image"}, + "What do these images show?", + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "The first image shows the statue of Liberty in New York. The second image picture depicts Idefix, the dog of Obelix in Asterix and Obelix.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "And who is that?"}]}, + ] + processor = self.get_processor() + # Make short sequence length to test that the fake tokens are added correctly + rendered = processor.apply_chat_template(messages, add_generation_prompt=True) + + expected_rendered = ( + "<|im_start|>User: What do these images show?\n" + "Assistant: The first image shows the statue of Liberty in New York. The second image picture depicts Idefix, the dog of Obelix in Asterix and Obelix.\n" + "User: And who is that?\n" + "Assistant:" + ) + self.assertEqual(rendered, expected_rendered) + + @unittest.skip(reason="Broken from common. Fixing TODO @zucchini-nlp @molbap") + def test_chat_template_video_special_processing(self): + pass + + # Override as SmolVLMProcessor needs image tokens in prompts + def prepare_text_inputs(self, batch_size: Optional[int] = None): + if batch_size is None: + return "lower newer " + + if batch_size < 1: + raise ValueError("batch_size must be greater than 0") + + if batch_size == 1: + return ["lower newer "] + return ["lower newer ", " upper older longer string"] + [" lower newer"] * ( + batch_size - 2 + ) + + # Override tests as inputs_ids padded dimension is the second one but not the last one + @require_vision + @require_torch + def test_kwargs_overrides_default_tokenizer_kwargs(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer", max_length=30) + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + input_str = self.prepare_text_inputs() + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input, return_tensors="pt", max_length=30) + self.assertEqual(len(inputs["input_ids"][0]), 30) + + @require_torch + @require_vision + def test_structured_kwargs_nested(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer") + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + + input_str = self.prepare_text_inputs() + image_input = self.prepare_image_inputs() + + # Define the kwargs for each modality + inputs = processor( + text=input_str, + images=image_input, + common_kwargs={"return_tensors": "pt"}, + images_kwargs={"max_image_size": {"longest_edge": 32}}, + text_kwargs={"padding": "max_length", "max_length": 120, "truncation": "longest_first"}, + ) + self.skip_processor_without_typed_kwargs(processor) + + self.assertEqual(inputs["pixel_values"].shape[3], 32) + + self.assertEqual(len(inputs["input_ids"][0]), 120) + + @require_torch + @require_vision + def test_structured_kwargs_nested_from_dict(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer") + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + input_str = self.prepare_text_inputs() + image_input = self.prepare_image_inputs() + + # Define the kwargs for each modality + all_kwargs = { + "common_kwargs": {"return_tensors": "pt"}, + "images_kwargs": {"max_image_size": {"longest_edge": 32}}, + "text_kwargs": {"padding": "max_length", "max_length": 120, "truncation": "longest_first"}, + } + + inputs = processor(text=input_str, images=image_input, **all_kwargs) + self.assertEqual(inputs["pixel_values"].shape[3], 32) + self.assertEqual(len(inputs["input_ids"][0]), 120) + + @require_vision + @require_torch + def test_tokenizer_defaults_preserved_by_kwargs(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer", max_length=30) + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + input_str = self.prepare_text_inputs() + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input, return_tensors="pt") + self.assertEqual(len(inputs["input_ids"][0]), 30) + + @require_torch + @require_vision + def test_unstructured_kwargs_batched(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer") + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + + input_str = self.prepare_text_inputs(batch_size=2) + image_input = self.prepare_image_inputs(batch_size=2) + image_input = [[image_input[0]], [image_input[1]]] + inputs = processor( + text=input_str, + images=image_input, + return_tensors="pt", + padding="longest", + max_length=76, + truncation=True, + max_image_size={"longest_edge": 30}, + ) + + self.assertEqual(inputs["pixel_values"].shape[2], 3) + self.assertEqual(inputs["pixel_values"].shape[3], 30) + self.assertEqual(len(inputs["input_ids"][0]), 76) + + @require_torch + @require_vision + def test_unstructured_kwargs(self): + if "image_processor" not in self.processor_class.attributes: + self.skipTest(f"image_processor attribute not present in {self.processor_class}") + image_processor = self.get_component("image_processor") + tokenizer = self.get_component("tokenizer") + + processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) + self.skip_processor_without_typed_kwargs(processor) + + input_str = self.prepare_text_inputs() + image_input = self.prepare_image_inputs() + inputs = processor( + text=input_str, + images=image_input, + return_tensors="pt", + max_image_size={"longest_edge": 32}, + padding="max_length", + max_length=120, + truncation="longest_first", + ) + + self.assertEqual(inputs["pixel_values"].shape[3], 32) + self.assertEqual(len(inputs["input_ids"][0]), 120) + + @require_torch + @require_vision + def test_text_only_inference(self): + """Test that the processor works correctly with text-only input.""" + processor_components = self.prepare_components() + processor_components["tokenizer"] = self.get_component("tokenizer", padding_side="left") + processor_kwargs = self.prepare_processor_dict() + + processor = self.processor_class(**processor_components, **processor_kwargs) + + text = "This is a simple text without images." + inputs = processor(text=text) + + tokenized_sentence = processor.tokenizer(text, add_special_tokens=False) + expected_input_ids = [tokenized_sentence["input_ids"]] + + self.assertEqual(inputs["input_ids"], expected_input_ids) + self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])]) + self.assertTrue("pixel_values" not in inputs) + self.assertTrue("pixel_attention_mask" not in inputs) + + # Test batch of texts without image tokens + texts = ["First text.", "Second piece of text."] + batch_inputs = processor(text=texts, padding=True) + + tokenized_1 = processor.tokenizer(texts[0], add_special_tokens=False) + tokenized_2 = processor.tokenizer(texts[1], add_special_tokens=False) + + expected_1 = tokenized_1["input_ids"] + expected_2 = tokenized_2["input_ids"] + + # Pad the shorter sequence + pad_len = len(expected_2) - len(expected_1) + if pad_len > 0: + padded_expected_1 = [self.padding_token_id] * pad_len + expected_1 + expected_attention_1 = [0] * pad_len + [1] * len(expected_1) + self.assertEqual(batch_inputs["input_ids"], [padded_expected_1, expected_2]) + self.assertEqual(batch_inputs["attention_mask"], [expected_attention_1, [1] * len(expected_2)]) + else: + pad_len = -pad_len + padded_expected_2 = [self.padding_token_id] * pad_len + expected_2 + expected_attention_2 = [0] * pad_len + [1] * len(expected_2) + self.assertEqual(batch_inputs["input_ids"], [expected_1, padded_expected_2]) + self.assertEqual(batch_inputs["attention_mask"], [[1] * len(expected_1), expected_attention_2]) + + @require_torch + @require_vision + def test_missing_images_error(self): + """Test that appropriate error is raised when images are referenced but not provided.""" + processor = self.get_processor() + + # Test single text with image token but no image + text = "Let me show you this image: What do you think?" + with self.assertRaises(ValueError) as context: + processor(text=text) + self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) + + # Test batch with image tokens but no images + texts = [ + "First text with token.", + "Second text with token.", + ] + with self.assertRaises(ValueError) as context: + processor(text=texts) + self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) + + # Test with None as Images + with self.assertRaises(ValueError) as context: + processor(text=text, images=None) + self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) + + with self.assertRaises(ValueError) as context: + processor(text=texts, images=None) + self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) From c90b8561f8a4a38b21398ac02f92fb09a18b92fe Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 10 Mar 2025 19:15:42 +0000 Subject: [PATCH 29/48] make style --- tests/models/phi3_v/test_image_processing_phi3_v.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/models/phi3_v/test_image_processing_phi3_v.py b/tests/models/phi3_v/test_image_processing_phi3_v.py index 396db35651b0..dd48a6140dd9 100644 --- a/tests/models/phi3_v/test_image_processing_phi3_v.py +++ b/tests/models/phi3_v/test_image_processing_phi3_v.py @@ -54,7 +54,7 @@ def __init__( image_std=[0.26862954, 0.26130258, 0.27577711], do_convert_rgb=True, do_pad=True, - num_crops=1 + num_crops=1, ): self.size = size if size is not None else {"longest_edge": max_resolution} self.parent = parent @@ -87,7 +87,7 @@ def prepare_image_processor_dict(self): "image_mean": self.image_mean, "image_std": self.image_std, "do_pad": self.do_pad, - "num_crops": self.num_crops + "num_crops": self.num_crops, } def get_expected_values(self, image_inputs, batched=False): @@ -99,9 +99,7 @@ def get_expected_values(self, image_inputs, batched=False): def expected_output_image_shape(self, images): height, width = self.get_expected_values(images, batched=True) - effective_nb_images = ( - (self.num_crops + 1) * self.num_images - ) + effective_nb_images = (self.num_crops + 1) * self.num_images return effective_nb_images, self.num_channels, height, width def prepare_image_inputs( From 1e797ccd2c6e717494eed0c405c8013f1f97205f Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 10 Mar 2025 19:22:18 +0000 Subject: [PATCH 30/48] fill docstring --- src/transformers/models/phi3_v/image_processing_phi3_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/image_processing_phi3_v.py b/src/transformers/models/phi3_v/image_processing_phi3_v.py index b37330eb12f3..509107315cc6 100644 --- a/src/transformers/models/phi3_v/image_processing_phi3_v.py +++ b/src/transformers/models/phi3_v/image_processing_phi3_v.py @@ -145,7 +145,7 @@ class Phi3VImageProcessor(BaseImageProcessor): Args: num_crops (`int`, *optional*, defaults to 1): Number of different image crops to use. - num_img_tokens (`int`, *optional*, defaults to 144): + num_img_tokens (`int`, *optional*, defaults to 144): Number of tokens to generate per image image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): Mean to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. From a68eff14ec87f47058b06e84356409ae825bf2c0 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 10 Mar 2025 19:32:31 +0000 Subject: [PATCH 31/48] Add chat template init --- src/transformers/models/phi3_v/processing_phi3_v.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py index 23dffdfb2b43..00bcab37fb6c 100644 --- a/src/transformers/models/phi3_v/processing_phi3_v.py +++ b/src/transformers/models/phi3_v/processing_phi3_v.py @@ -43,18 +43,22 @@ class Phi3VProcessor(ProcessorMixin): The image processor is a required input. tokenizer ([`LlamaTokenizerFast`]): The tokenizer is a required input. + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. """ attributes = ["image_processor", "tokenizer"] + valid_kwargs = ["chat_template"] image_processor_class = "Phi3VImageProcessor" tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") special_image_token = "<|image|>" - def __init__(self, image_processor, tokenizer): + def __init__(self, image_processor, tokenizer, chat_template: str = None, **kwargs): self.image_processor = image_processor self.tokenizer = tokenizer self.num_img_tokens = image_processor.num_img_tokens self.img_tokens = [f"<|image_{i+1}|>" for i in range(1000000)] + super().__init__(image_processor, tokenizer, chat_template=chat_template, **kwargs) def __call__( self, From 9bbfb0686e3d316c827437b8d073c66a832d76dc Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Fri, 14 Mar 2025 01:39:50 -0700 Subject: [PATCH 32/48] Small adjustments to test processor --- tests/models/phi3_v/test_processor_phi3_v.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index dae0a4c0896f..41b0fdd3f779 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -63,16 +63,9 @@ def setUpClass(cls): ) ) cls.bos_token = processor.tokenizer.bos_token - cls.image_token = processor.image_token - cls.fake_image_token = processor.fake_image_token - cls.global_img_token = processor.global_image_token cls.bos_token_id = processor.tokenizer.convert_tokens_to_ids(cls.bos_token) - cls.image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.image_token) - cls.fake_image_token_id = processor.tokenizer.convert_tokens_to_ids(cls.fake_image_token) - cls.global_img_tokens_id = processor.tokenizer(cls.global_img_token, add_special_tokens=False)["input_ids"] cls.padding_token_id = processor.tokenizer.pad_token_id - cls.image_seq_len = processor.image_seq_len def get_tokenizer(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer From eaace73faa78cf3ead6e3b5ca9ccde357498ff9a Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 14 Mar 2025 17:37:39 +0000 Subject: [PATCH 33/48] Stop trying to use a slow processor bc it's cursed --- .../models/phi3_v/processing_phi3_v.py | 20 ++++++++++++++++--- tests/models/phi3_v/test_processor_phi3_v.py | 1 - 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/phi3_v/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py index 00bcab37fb6c..6745feeab39e 100644 --- a/src/transformers/models/phi3_v/processing_phi3_v.py +++ b/src/transformers/models/phi3_v/processing_phi3_v.py @@ -50,7 +50,7 @@ class Phi3VProcessor(ProcessorMixin): attributes = ["image_processor", "tokenizer"] valid_kwargs = ["chat_template"] image_processor_class = "Phi3VImageProcessor" - tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + tokenizer_class = "LlamaTokenizerFast" special_image_token = "<|image|>" def __init__(self, image_processor, tokenizer, chat_template: str = None, **kwargs): @@ -68,6 +68,7 @@ def __call__( truncation: Union[bool, str, TruncationStrategy] = None, max_length=None, return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH, + add_special_tokens: bool = True, ) -> BatchFeature: """ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` @@ -125,6 +126,7 @@ def __call__( truncation=truncation, max_length=max_length, return_tensors=return_tensors, + add_special_tokens=add_special_tokens, ) return inputs @@ -155,11 +157,23 @@ def get_special_image_token_id(self): return self.tokenizer.convert_tokens_to_ids(self.special_image_token) def _convert_images_texts_to_inputs( - self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None + self, + images, + texts, + padding=False, + truncation=None, + max_length=None, + return_tensors=None, + add_special_tokens=True, ): if not len(images): model_inputs = self.tokenizer( - texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length + texts, + return_tensors=return_tensors, + padding=padding, + truncation=truncation, + max_length=max_length, + add_special_tokens=add_special_tokens, ) return BatchFeature(data={**model_inputs}) diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index 41b0fdd3f779..723ae583344c 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -78,7 +78,6 @@ def get_processor(self, **kwargs): def prepare_processor_dict(self): return { - "image_seq_len": self.image_seq_len, "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %}\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", } From e5e2649b09cf43b98482512074cad60241cac998 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Tue, 1 Apr 2025 02:02:58 -0700 Subject: [PATCH 34/48] test processor updates and slight changes to processing --- .../models/phi3_v/processing_phi3_v.py | 8 - tests/models/phi3_v/test_processor_phi3_v.py | 274 ++---------------- 2 files changed, 26 insertions(+), 256 deletions(-) diff --git a/src/transformers/models/phi3_v/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py index 6745feeab39e..36e644275b66 100644 --- a/src/transformers/models/phi3_v/processing_phi3_v.py +++ b/src/transformers/models/phi3_v/processing_phi3_v.py @@ -51,7 +51,6 @@ class Phi3VProcessor(ProcessorMixin): valid_kwargs = ["chat_template"] image_processor_class = "Phi3VImageProcessor" tokenizer_class = "LlamaTokenizerFast" - special_image_token = "<|image|>" def __init__(self, image_processor, tokenizer, chat_template: str = None, **kwargs): self.image_processor = image_processor @@ -149,13 +148,6 @@ def calc_num_image_tokens_from_image_size(self, width, height): """ return self.image_processor.calc_num_image_tokens_from_image_size(width, height) - @property - def special_image_token_id(self): - return self.tokenizer.convert_tokens_to_ids(self.special_image_token) - - def get_special_image_token_id(self): - return self.tokenizer.convert_tokens_to_ids(self.special_image_token) - def _convert_images_texts_to_inputs( self, images, diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index 723ae583344c..e501f93ae91c 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -81,181 +81,38 @@ def prepare_processor_dict(self): "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %}\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", } - def get_split_image_expected_tokens(self, processor, image_rows, image_cols): - text_split_images = [] - for n_h in range(image_rows): - for n_w in range(image_cols): - text_split_images += ( - [self.fake_image_token_id] - + processor.tokenizer(f"", add_special_tokens=False)["input_ids"] - + [self.image_token_id] * self.image_seq_len - ) - text_split_images += processor.tokenizer("\n", add_special_tokens=False)["input_ids"] - text_split_images = text_split_images[:-1] # remove last newline - # add double newline, as it gets its own token - text_split_images += processor.tokenizer("\n\n", add_special_tokens=False)["input_ids"] - text_split_images += ( - [self.fake_image_token_id] - + self.global_img_tokens_id - + [self.image_token_id] * self.image_seq_len - + [self.fake_image_token_id] - ) - return text_split_images + def get_expected_input_ids(self, processor, image): + num_crops = processor.image_processor.num_crops + imgsize = list(image.size) + imgsize.sort() + height, width = imgsize + ratio = width / height + scale = 1 + while scale * np.ceil(scale / ratio) <= num_crops: + scale += 1 + scale -= 1 + num_img_tokens = (scale * scale + 1) * 144 + 1 + (scale + 1) * 12 + img_tokens = [-1 for num in range(num_img_tokens)] + img_tokens.append(1) + return img_tokens @classmethod def tearDownClass(cls): shutil.rmtree(cls.tmpdirname) - def test_process_interleaved_images_prompts_no_image_splitting(self): - processor_components = self.prepare_components() - processor_components["tokenizer"] = self.get_component("tokenizer", padding_side="left") - processor_components["image_processor"] = self.get_component("image_processor", do_image_splitting=False) - processor_kwargs = self.prepare_processor_dict() - - processor = self.processor_class(**processor_components, **processor_kwargs) - - # Test that a single image is processed correctly - inputs = processor(images=self.image1) - image1_expected_size = (512, 512) - self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 1, 3, *image1_expected_size)) - self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 1, *image1_expected_size)) - # fmt: on - - # Test a single sample with image and text - image_str = "" - text_str = "In this image, we see" - text = image_str + text_str - inputs = processor(text=text, images=self.image1) - - # fmt: off - tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) - expected_input_ids = [[self.fake_image_token_id] + self.global_img_tokens_id + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] + tokenized_sentence["input_ids"]] - self.assertEqual(inputs["input_ids"], expected_input_ids) - self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])]) - self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 1, 3, *image1_expected_size)) - self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 1, *image1_expected_size)) - # fmt: on - - # Test that batch is correctly processed - image_str = "" - text_str_1 = "In this image, we see" - text_str_2 = "In this image, we see" - - text = [ - image_str + text_str_1, - image_str + image_str + text_str_2, - ] - images = [[self.image1], [self.image2, self.image3]] - - inputs = processor(text=text, images=images, padding=True) - - # fmt: off - tokenized_sentence_1 = processor.tokenizer(text_str_1, add_special_tokens=False) - tokenized_sentence_2 = processor.tokenizer(text_str_2, add_special_tokens=False) - image_tokens = [self.fake_image_token_id] + self.global_img_tokens_id + [self.image_token_id] * self.image_seq_len + [self.fake_image_token_id] - expected_input_ids_1 = image_tokens + tokenized_sentence_1["input_ids"] - expected_input_ids_2 = 2 * image_tokens + tokenized_sentence_2["input_ids"] - # Pad the first input to match the second input - pad_len = len(expected_input_ids_2) - len(expected_input_ids_1) - padded_expected_input_ids_1 = [self.padding_token_id] * pad_len + expected_input_ids_1 - - self.assertEqual( - inputs["input_ids"], [padded_expected_input_ids_1, expected_input_ids_2] - ) - self.assertEqual( - inputs["attention_mask"], - [[0] * pad_len + [1] * len(expected_input_ids_1), [1] * len(expected_input_ids_2)] - ) - self.assertEqual(np.array(inputs['pixel_values']).shape, (2, 2, 3, 512, 512)) - self.assertEqual(np.array(inputs['pixel_attention_mask']).shape, (2, 2, 512, 512)) - # fmt: on - - def test_process_interleaved_images_prompts_image_splitting(self): - processor_components = self.prepare_components() - processor_components["tokenizer"] = self.get_component("tokenizer", padding_side="left") - processor_components["image_processor"] = self.get_component("image_processor", do_image_splitting=True) - processor_kwargs = self.prepare_processor_dict() - - processor = self.processor_class(**processor_components, **processor_kwargs) - - # Test that a single image is processed correctly - inputs = processor(images=self.image1) - self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 13, 3, 512, 512)) - self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 13, 512, 512)) - # fmt: on - self.maxDiff = None - - # Test a single sample with image and text - image_str = "" - text_str = "In this image, we see" - text = image_str + text_str - inputs = processor(text=text, images=self.image1) - - # fmt: off - tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) - split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) - expected_input_ids_1 = [split_image1_tokens + tokenized_sentence["input_ids"]] - self.assertEqual(inputs["input_ids"], expected_input_ids_1) - self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids_1[0])]) - self.assertEqual(np.array(inputs["pixel_values"]).shape, (1, 13, 3, 512, 512)) - self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (1, 13, 512, 512)) - # fmt: on - - # Test that batch is correctly processed - image_str = "" - text_str_1 = "In this image, we see" - text_str_2 = "bla, bla" - - text = [ - image_str + text_str_1, - text_str_2 + image_str + image_str, - ] - images = [[self.image1], [self.image2, self.image3]] - - inputs = processor(text=text, images=images, padding=True) - - # fmt: off - tokenized_sentence_1 = processor.tokenizer(text_str_1, add_special_tokens=False) - tokenized_sentence_2 = processor.tokenizer(text_str_2, add_special_tokens=False) - - split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) - split_image2_tokens = self.get_split_image_expected_tokens(processor, 4, 4) - split_image3_tokens = self.get_split_image_expected_tokens(processor, 3, 4) - expected_input_ids_1 = split_image1_tokens + tokenized_sentence_1["input_ids"] - expected_input_ids_2 = tokenized_sentence_2["input_ids"] + split_image2_tokens + split_image3_tokens - # Pad the first input to match the second input - pad_len = len(expected_input_ids_2) - len(expected_input_ids_1) - padded_expected_input_ids_1 = [self.padding_token_id] * pad_len + expected_input_ids_1 - - self.assertEqual( - inputs["input_ids"], [padded_expected_input_ids_1, expected_input_ids_2] - ) - self.assertEqual( - inputs["attention_mask"], - [[0] * pad_len + [1] * len(expected_input_ids_1), [1] * len(expected_input_ids_2)] - ) - self.assertEqual(np.array(inputs['pixel_values']).shape, (2, 30, 3, 512, 512)) - self.assertEqual(np.array(inputs['pixel_attention_mask']).shape, (2, 30, 512, 512)) - # fmt: on - - def test_add_special_tokens_processor(self): + def test_image_token_count(self): processor = self.get_processor() - image_str = "" - text_str = "In this image, we see" + image_str = "<|image_1|>" + text_str = "In this image, we see\n" text = text_str + image_str - # fmt: off - inputs = processor(text=text, images=self.image1, add_special_tokens=False) - tokenized_sentence = processor.tokenizer(text_str, add_special_tokens=False) - split_image1_tokens = self.get_split_image_expected_tokens(processor, 3, 4) + inputs = processor(text=text, images=self.image1, return_tensors='pt') + tokenized_sentence = processor.tokenizer(text_str) + split_image1_tokens = self.get_expected_input_ids(processor, self.image1) expected_input_ids = [tokenized_sentence["input_ids"] + split_image1_tokens] - self.assertEqual(inputs["input_ids"], expected_input_ids) - - inputs = processor(text=text, images=self.image1) - expected_input_ids = [tokenized_sentence["input_ids"] + split_image1_tokens] - self.assertEqual(inputs["input_ids"], expected_input_ids) - # fmt: on + inputs = inputs["input_ids"].tolist() + self.assertEqual(inputs, expected_input_ids) @unittest.skip(reason="from @molbap @zucchini-nlp, passing non-nested images is error-prone and not recommended") def test_non_nested_images_with_batched_text(self): @@ -277,55 +134,6 @@ def test_non_nested_images_with_batched_text(self): self.assertEqual(np.array(inputs["pixel_values"]).shape, (2, 2, 3, 512, 512)) self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (2, 2, 512, 512)) - # Copied from tests.models.idefics2.test_processor_idefics2.Idefics2ProcessorTest.test_process_interleaved_images_prompts_image_error - def test_process_interleaved_images_prompts_image_error(self): - processor = self.get_processor() - - text = [ - "This is a test sentence.", - "In this other sentence we try some good things", - ] - images = [[self.image1], [self.image2]] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - - text = [ - "This is a test sentence.", - "In this other sentence we try some good things", - ] - images = [[self.image1], [self.image2, self.image3]] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [self.image1, self.image2, self.image3] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - - text = [ - "This is a test sentence.", - "In this other sentence we try some good things", - ] - images = [[self.image1], []] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [[], [self.image2]] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [self.image1, self.image2] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - images = [self.image1] - with self.assertRaises(ValueError): - processor(text=text, images=images, padding=True) - def test_apply_chat_template(self): # Message contains content which a mix of lists with images and image urls and string messages = [ @@ -530,19 +338,19 @@ def test_text_only_inference(self): processor = self.processor_class(**processor_components, **processor_kwargs) text = "This is a simple text without images." - inputs = processor(text=text) + inputs = processor(text=text, add_special_tokens=False, return_tensors=None) tokenized_sentence = processor.tokenizer(text, add_special_tokens=False) - expected_input_ids = [tokenized_sentence["input_ids"]] + expected_input_ids = tokenized_sentence["input_ids"] self.assertEqual(inputs["input_ids"], expected_input_ids) - self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids[0])]) + self.assertEqual(inputs["attention_mask"], [[1] * len(expected_input_ids)][0]) self.assertTrue("pixel_values" not in inputs) self.assertTrue("pixel_attention_mask" not in inputs) # Test batch of texts without image tokens texts = ["First text.", "Second piece of text."] - batch_inputs = processor(text=texts, padding=True) + batch_inputs = processor(text=texts, padding=True, add_special_tokens=False, return_tensors=None) tokenized_1 = processor.tokenizer(texts[0], add_special_tokens=False) tokenized_2 = processor.tokenizer(texts[1], add_special_tokens=False) @@ -563,33 +371,3 @@ def test_text_only_inference(self): expected_attention_2 = [0] * pad_len + [1] * len(expected_2) self.assertEqual(batch_inputs["input_ids"], [expected_1, padded_expected_2]) self.assertEqual(batch_inputs["attention_mask"], [[1] * len(expected_1), expected_attention_2]) - - @require_torch - @require_vision - def test_missing_images_error(self): - """Test that appropriate error is raised when images are referenced but not provided.""" - processor = self.get_processor() - - # Test single text with image token but no image - text = "Let me show you this image: What do you think?" - with self.assertRaises(ValueError) as context: - processor(text=text) - self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) - - # Test batch with image tokens but no images - texts = [ - "First text with token.", - "Second text with token.", - ] - with self.assertRaises(ValueError) as context: - processor(text=texts) - self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) - - # Test with None as Images - with self.assertRaises(ValueError) as context: - processor(text=text, images=None) - self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) - - with self.assertRaises(ValueError) as context: - processor(text=texts, images=None) - self.assertTrue("tokens in the text but no images/videos were passed" in str(context.exception)) From 85aab709c77cd18f813431df823c1147184cef3f Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Wed, 2 Apr 2025 14:22:50 -0700 Subject: [PATCH 35/48] removed unnecessary tests --- tests/models/phi3_v/test_processor_phi3_v.py | 213 ------------------- 1 file changed, 213 deletions(-) diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index e501f93ae91c..23f68f8f5896 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -114,219 +114,6 @@ def test_image_token_count(self): inputs = inputs["input_ids"].tolist() self.assertEqual(inputs, expected_input_ids) - @unittest.skip(reason="from @molbap @zucchini-nlp, passing non-nested images is error-prone and not recommended") - def test_non_nested_images_with_batched_text(self): - processor = self.get_processor() - processor.image_processor.do_image_splitting = False - - image_str = "" - text_str_1 = "In this image, we see" - text_str_2 = "In this image, we see" - - text = [ - image_str + text_str_1, - image_str + image_str + text_str_2, - ] - images = [[self.image1], [self.image2, self.image3]] - - inputs = processor(text=text, images=images, padding=True) - - self.assertEqual(np.array(inputs["pixel_values"]).shape, (2, 2, 3, 512, 512)) - self.assertEqual(np.array(inputs["pixel_attention_mask"]).shape, (2, 2, 512, 512)) - - def test_apply_chat_template(self): - # Message contains content which a mix of lists with images and image urls and string - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What do these images show?"}, - {"type": "image"}, - {"type": "image"}, - "What do these images show?", - ], - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "The first image shows the statue of Liberty in New York. The second image picture depicts Idefix, the dog of Obelix in Asterix and Obelix.", - } - ], - }, - {"role": "user", "content": [{"type": "text", "text": "And who is that?"}]}, - ] - processor = self.get_processor() - # Make short sequence length to test that the fake tokens are added correctly - rendered = processor.apply_chat_template(messages, add_generation_prompt=True) - - expected_rendered = ( - "<|im_start|>User: What do these images show?\n" - "Assistant: The first image shows the statue of Liberty in New York. The second image picture depicts Idefix, the dog of Obelix in Asterix and Obelix.\n" - "User: And who is that?\n" - "Assistant:" - ) - self.assertEqual(rendered, expected_rendered) - - @unittest.skip(reason="Broken from common. Fixing TODO @zucchini-nlp @molbap") - def test_chat_template_video_special_processing(self): - pass - - # Override as SmolVLMProcessor needs image tokens in prompts - def prepare_text_inputs(self, batch_size: Optional[int] = None): - if batch_size is None: - return "lower newer " - - if batch_size < 1: - raise ValueError("batch_size must be greater than 0") - - if batch_size == 1: - return ["lower newer "] - return ["lower newer ", " upper older longer string"] + [" lower newer"] * ( - batch_size - 2 - ) - - # Override tests as inputs_ids padded dimension is the second one but not the last one - @require_vision - @require_torch - def test_kwargs_overrides_default_tokenizer_kwargs(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer", max_length=30) - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - input_str = self.prepare_text_inputs() - image_input = self.prepare_image_inputs() - - inputs = processor(text=input_str, images=image_input, return_tensors="pt", max_length=30) - self.assertEqual(len(inputs["input_ids"][0]), 30) - - @require_torch - @require_vision - def test_structured_kwargs_nested(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer") - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - - input_str = self.prepare_text_inputs() - image_input = self.prepare_image_inputs() - - # Define the kwargs for each modality - inputs = processor( - text=input_str, - images=image_input, - common_kwargs={"return_tensors": "pt"}, - images_kwargs={"max_image_size": {"longest_edge": 32}}, - text_kwargs={"padding": "max_length", "max_length": 120, "truncation": "longest_first"}, - ) - self.skip_processor_without_typed_kwargs(processor) - - self.assertEqual(inputs["pixel_values"].shape[3], 32) - - self.assertEqual(len(inputs["input_ids"][0]), 120) - - @require_torch - @require_vision - def test_structured_kwargs_nested_from_dict(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer") - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - input_str = self.prepare_text_inputs() - image_input = self.prepare_image_inputs() - - # Define the kwargs for each modality - all_kwargs = { - "common_kwargs": {"return_tensors": "pt"}, - "images_kwargs": {"max_image_size": {"longest_edge": 32}}, - "text_kwargs": {"padding": "max_length", "max_length": 120, "truncation": "longest_first"}, - } - - inputs = processor(text=input_str, images=image_input, **all_kwargs) - self.assertEqual(inputs["pixel_values"].shape[3], 32) - self.assertEqual(len(inputs["input_ids"][0]), 120) - - @require_vision - @require_torch - def test_tokenizer_defaults_preserved_by_kwargs(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer", max_length=30) - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - input_str = self.prepare_text_inputs() - image_input = self.prepare_image_inputs() - - inputs = processor(text=input_str, images=image_input, return_tensors="pt") - self.assertEqual(len(inputs["input_ids"][0]), 30) - - @require_torch - @require_vision - def test_unstructured_kwargs_batched(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer") - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - - input_str = self.prepare_text_inputs(batch_size=2) - image_input = self.prepare_image_inputs(batch_size=2) - image_input = [[image_input[0]], [image_input[1]]] - inputs = processor( - text=input_str, - images=image_input, - return_tensors="pt", - padding="longest", - max_length=76, - truncation=True, - max_image_size={"longest_edge": 30}, - ) - - self.assertEqual(inputs["pixel_values"].shape[2], 3) - self.assertEqual(inputs["pixel_values"].shape[3], 30) - self.assertEqual(len(inputs["input_ids"][0]), 76) - - @require_torch - @require_vision - def test_unstructured_kwargs(self): - if "image_processor" not in self.processor_class.attributes: - self.skipTest(f"image_processor attribute not present in {self.processor_class}") - image_processor = self.get_component("image_processor") - tokenizer = self.get_component("tokenizer") - - processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor) - self.skip_processor_without_typed_kwargs(processor) - - input_str = self.prepare_text_inputs() - image_input = self.prepare_image_inputs() - inputs = processor( - text=input_str, - images=image_input, - return_tensors="pt", - max_image_size={"longest_edge": 32}, - padding="max_length", - max_length=120, - truncation="longest_first", - ) - - self.assertEqual(inputs["pixel_values"].shape[3], 32) - self.assertEqual(len(inputs["input_ids"][0]), 120) - @require_torch @require_vision def test_text_only_inference(self): From 5ba3fb93ae0d5428a8311b6eb7093ca5afd658eb Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Wed, 2 Apr 2025 14:53:37 -0700 Subject: [PATCH 36/48] maked fix-copies --- .../models/phi3_v/modeling_phi3_v.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 737112c27841..c9ea9a17a049 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -1712,29 +1712,28 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.model.embed_tokens = value + @can_return_tuple @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) def forward( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + past_key_values: Optional[Cache] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + ) -> SequenceClassifierOutputWithPast: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - model_outputs = self.model( + model_outputs: BaseModelOutputWithPast = self.model( input_ids, attention_mask=attention_mask, position_ids=position_ids, @@ -1743,9 +1742,8 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - return_dict=return_dict, ) - hidden_states = model_outputs[0] + hidden_states = model_outputs.last_hidden_state logits = self.score(hidden_states) if input_ids is not None: @@ -1760,7 +1758,7 @@ def forward( elif input_ids is not None: # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) - token_indices = torch.arange(input_ids.shape[-1], device=logits.device) + token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32) last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) else: last_non_pad_token = -1 @@ -1775,10 +1773,6 @@ def forward( if labels is not None: loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) - if not return_dict: - output = (pooled_logits,) + model_outputs[1:] - return ((loss,) + output) if loss is not None else output - return SequenceClassifierOutputWithPast( loss=loss, logits=pooled_logits, From 9e0101865a2ed0441877542462afe6624c12b2bb Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Wed, 2 Apr 2025 14:58:44 -0700 Subject: [PATCH 37/48] maked fix-copies --- src/transformers/models/phi3_v/modeling_phi3_v.py | 1 + src/transformers/models/phi3_v/processing_phi3_v.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index c9ea9a17a049..9ae98f008f24 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -41,6 +41,7 @@ add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, + can_return_tuple, is_flash_attn_greater_or_equal_2_10, logging, replace_return_docstrings, diff --git a/src/transformers/models/phi3_v/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py index 36e644275b66..57a9fd421c45 100644 --- a/src/transformers/models/phi3_v/processing_phi3_v.py +++ b/src/transformers/models/phi3_v/processing_phi3_v.py @@ -45,6 +45,7 @@ class Phi3VProcessor(ProcessorMixin): The tokenizer is a required input. chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string. + chat_template (`str`, *optional*): """ attributes = ["image_processor", "tokenizer"] From dcb8e2738f0435d9b73a7bd853ffe7a19f9fa610 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sat, 5 Apr 2025 17:44:38 -0700 Subject: [PATCH 38/48] Modular file added --- .../models/phi3_v/modeling_phi3_v.py | 219 +- .../models/phi3_v/modular_phi3_v.py | 1790 +++++++++++++++++ 2 files changed, 1848 insertions(+), 161 deletions(-) create mode 100644 src/transformers/models/phi3_v/modular_phi3_v.py diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 9ae98f008f24..8f0962ad5aa8 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -1,33 +1,21 @@ -# coding=utf-8 -# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""PyTorch Phi-3-V model.""" - -import inspect +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/phi3_v/modular_phi3_v.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_phi3_v.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 import math import warnings from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F -import torch.utils.checkpoint from torch import nn from torch.nn import CrossEntropyLoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache +from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask from ...modeling_outputs import ( @@ -37,6 +25,9 @@ TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel +from ...models.clip.configuration_clip import CLIPVisionConfig +from ...models.clip.modeling_clip import CLIPAttention, CLIPVisionModel +from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv from ...utils import ( add_code_sample_docstrings, add_start_docstrings, @@ -49,41 +40,10 @@ from .configuration_phi3_v import Phi3VConfig -try: - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) -except ImportError: - pass - -import torch - -from ... import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig -from ...models.clip.modeling_clip import CLIPAttention - - logger = logging.get_logger(__name__) - -MAX_INPUT_ID = int(1e9) - -CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( - attention_dropout=0.0, - dropout=0.0, - hidden_act="quick_gelu", - hidden_size=1024, - image_size=336, - initializer_factor=1.0, - initializer_range=0.02, - intermediate_size=4096, - layer_norm_eps=1e-05, - num_attention_heads=16, - num_channels=3, - num_hidden_layers=24, - patch_size=14, - projection_dim=768, -) +_CHECKPOINT_FOR_DOC = "microsoft/Phi-3.5-vision-instruct" +_CONFIG_FOR_DOC = "Phi3VConfig" class CLIPAttentionFA2(CLIPAttention): @@ -120,6 +80,26 @@ def forward( return attn_output, None +MAX_INPUT_ID = int(1e9) + +CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( + attention_dropout=0.0, + dropout=0.0, + hidden_act="quick_gelu", + hidden_size=1024, + image_size=336, + initializer_factor=1.0, + initializer_range=0.02, + intermediate_size=4096, + layer_norm_eps=1e-05, + num_attention_heads=16, + num_channels=3, + num_hidden_layers=24, + patch_size=14, + projection_dim=768, +) + + class Phi3ImageEmbedding(nn.Module): """Phi3 Image embedding.""" @@ -163,9 +143,9 @@ def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None: self.with_learnable_separator = kwargs.get("with_learnable_separator", False) self.hd_transform_order = kwargs.get("hd_transform_order", "glb_sub") # with_hd_transform and with_learnable_separator should have same value - assert ( - self.use_hd_transform == self.with_learnable_separator - ), "use_hd_transform and with_learnable_separator should have same value" + assert self.use_hd_transform == self.with_learnable_separator, ( + "use_hd_transform and with_learnable_separator should have same value" + ) if self.with_learnable_separator: assert self.use_hd_transform, "learnable separator is only for hd transform" # 1024 * 4, merge spatial to channel dimension @@ -317,27 +297,6 @@ def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): .reshape(num_images, h_crop * H // 2, w_crop * H // 2, 4 * C) # n_img, h_crop*12, w_crop*12, 4096 ) - # alternative implementation using einops - # from einops import rearrange - # image_features_nhwc = rearrange( - # image_features, - # 'N (H W) c -> N H W c', - # H=H, - # W=H, - # ) - # image_features_2x2merge = rearrange( - # image_features_nhwc, - # 'N (h h_pool) (w w_pool) c -> N h w (h_pool w_pool c)', - # h_pool=2, - # w_pool=2, - # ) - # image_features_hd = rearrange( - # image_features_2x2merge, - # '(n_img h_crop w_crop) h w C -> n_img (h_crop h) (w_crop w) C', - # h_crop=h_crop, - # w_crop=w_crop, - # ) - return image_features_hd def add_image_newline(self, image_features_hd): @@ -354,17 +313,6 @@ def add_image_newline(self, image_features_hd): return image_features_hd_newline -logger = logging.get_logger(__name__) - -_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-vision-128k-instruct" -_CONFIG_FOR_DOC = "Phi3VConfig" - -PHI3V_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "microsoft/Phi-3-vision-128k-instruct", - # See all Phi-3 models at https://huggingface.co/models?filter=Phi-3 -] - - class Phi3RMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): """ @@ -382,19 +330,6 @@ def forward(self, hidden_states): return self.weight * hidden_states.to(input_dtype) -def _get_unpad_data(attention_mask): - seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) - indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() - max_seqlen_in_batch = seqlens_in_batch.max().item() - cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) - return ( - indices, - cu_seqlens, - max_seqlen_in_batch, - ) - - -# TODO Used to be copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding, but the signature changed class Phi3RotaryEmbedding(nn.Module): def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): super().__init__() @@ -507,42 +442,6 @@ def forward(self, x, position_ids, seq_len=None): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -# Copied from transformers.models.llama.modeling_llama.rotate_half -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb -def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`, *optional*): - Deprecated and unused. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - class Phi3MLP(nn.Module): def __init__(self, config): super().__init__() @@ -562,19 +461,6 @@ def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: return self.down_proj(up_states) -# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """ - This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, - num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) - """ - batch, num_key_value_heads, slen, head_dim = hidden_states.shape - if n_rep == 1: - return hidden_states - hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) - return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) - - class Phi3Attention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" @@ -710,6 +596,18 @@ def forward( return attn_output, attn_weights, past_key_value +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + class Phi3FlashAttention2(Phi3Attention): """ Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays @@ -1016,8 +914,6 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query ) -# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3V -# TODO @Arthur no longer copied from LLama after static cache class Phi3SdpaAttention(Phi3Attention): """ Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from @@ -1503,7 +1399,6 @@ def forward( class Phi3VForCausalLM(Phi3VPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3V def __init__(self, config): super().__init__(config) self.model = Phi3VModel(config) @@ -1513,31 +1408,24 @@ def __init__(self, config): # Initialize weights and apply final processing self.post_init() - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings def get_input_embeddings(self): return self.model.embed_tokens - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings def set_input_embeddings(self, value): self.model.embed_tokens = value - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings def get_output_embeddings(self): return self.lm_head - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder def set_decoder(self, decoder): self.model = decoder - # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder def get_decoder(self): return self.model - # Ignore copy @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( @@ -1696,7 +1584,6 @@ def _reorder_cache(past_key_values, beam_idx): """, PHI3V_START_DOCSTRING, ) -# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3V, LLAMA->PHI3V, self.transformer->self.model, transformer_outputs->model_outputs class Phi3VForSequenceClassification(Phi3VPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1790,7 +1677,6 @@ def forward( """, PHI3V_START_DOCSTRING, ) -# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3V,MPT->PHI3V,self.transformer->self.model,transformer_outputs->model_outputs class Phi3VForTokenClassification(Phi3VPreTrainedModel): def __init__(self, config: Phi3VConfig): super().__init__(config) @@ -1874,9 +1760,20 @@ def forward( __all__ = [ + "CLIPAttentionFA2", + "Phi3Attention", + "Phi3DecoderLayer", + "Phi3FlashAttention2", + "Phi3ImageEmbedding", + "Phi3MLP", + "Phi3RMSNorm", + "Phi3RotaryEmbedding", + "Phi3SdpaAttention", + "Phi3SuScaledRotaryEmbedding", "Phi3VForCausalLM", "Phi3VForSequenceClassification", "Phi3VForTokenClassification", "Phi3VModel", "Phi3VPreTrainedModel", + "Phi3YarnScaledRotaryEmbedding", ] diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py new file mode 100644 index 000000000000..e0a159c4a940 --- /dev/null +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -0,0 +1,1790 @@ +"""PyTorch Phi-3-V model.""" + +import inspect +import math +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask +from ...modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + can_return_tuple, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_phi3_v import Phi3VConfig + + +try: + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) +except ImportError: + pass + +import torch + +from ...configuration_utils import PretrainedConfig +from ...models.clip.configuration_clip import CLIPVisionConfig +from ...models.clip.modeling_clip import CLIPAttention, CLIPVisionModel +from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, rotate_half + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "microsoft/Phi-3.5-vision-instruct" +_CONFIG_FOR_DOC = "Phi3VConfig" + +PHI3V_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "microsoft/Phi-3-vision-128k-instruct", + # See all Phi-3 models at https://huggingface.co/models?filter=Phi-3 +] + +MAX_INPUT_ID = int(1e9) + +CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( + attention_dropout=0.0, + dropout=0.0, + hidden_act="quick_gelu", + hidden_size=1024, + image_size=336, + initializer_factor=1.0, + initializer_range=0.02, + intermediate_size=4096, + layer_norm_eps=1e-05, + num_attention_heads=16, + num_channels=3, + num_hidden_layers=24, + patch_size=14, + projection_dim=768, +) + +PHI3_ATTENTION_CLASSES = { + "eager": Phi3Attention, + "flash_attention_2": Phi3FlashAttention2, + "sdpa": Phi3SdpaAttention, +} + +class CLIPAttentionFA2(CLIPAttention): + """Add flash attention 2 to CLIPAttention. (This is only used in the vision encoder)""" + + def forward( + self, + hidden_states, + attention_mask=None, + causal_attention_mask=None, + output_attentions=False, + ): + """Input shape: Batch x Time x Channel""" + + assert attention_mask is None, "CLIPAttentionFA2 does not support attention_mask" + assert causal_attention_mask is None, "CLIPAttentionFA2 does not support causal_attention_mask" + assert output_attentions is False, "CLIPAttentionFA2 does not support output_attentions" + + bsz, tgt_len, embed_dim = hidden_states.size() + query_states = self.q_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + key_states = self.k_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + value_states = self.v_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim) + + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout_p=self.dropout if self.training else 0.0, + softmax_scale=self.scale, + causal=False, + ).reshape(bsz, tgt_len, embed_dim) + + attn_output = self.out_proj(attn_output) + return attn_output, None + + +class Phi3ImageEmbedding(nn.Module): + """Phi3 Image embedding.""" + + def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None: + super().__init__() + + # n_embed or hidden_size + hidden_size = config.n_embd if hasattr(config, "n_embd") else config.hidden_size + if hasattr(config, "embd_pdrop") or hasattr(config, "embed_pdrop"): + embd_drop = config.embd_pdrop if hasattr(config, "embd_pdrop") else config.embed_pdrop + self.drop = nn.Dropout(embd_drop) + else: + self.drop = None + + self.wte = wte + + if isinstance(config.img_processor, dict) and config.img_processor.get("name", None) == "clip_vision_model": + assert "model_name" in config.img_processor, "model_name must be provided for CLIPVisionModel" + assert "image_dim_out" in config.img_processor, "image_dim_out must be provided for CLIPVisionModel" + assert "num_img_tokens" in config.img_processor, "num_img_tokens must be provided for CLIPVisionModel" + assert config.img_processor["model_name"] == "openai/clip-vit-large-patch14-336" + clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG + self.img_processor = CLIPVisionModel(clip_config) + image_dim_out = config.img_processor["image_dim_out"] + self.num_img_tokens = config.img_processor["num_img_tokens"] + + # FA2 in CLIP + if config._attn_implementation == "flash_attention_2": + for layer in self.img_processor.vision_model.encoder.layers: + clip_fa2 = CLIPAttentionFA2(clip_config) + del layer.self_attn + layer.self_attn = clip_fa2 + else: + raise NotImplementedError(f"img_processor = {config.img_processor}, not implemented") + + self.image_dim_out = image_dim_out + self.img_sizes = None + + # global_gn and sub_gn for hd transform, serves as line separator + self.use_hd_transform = kwargs.get("use_hd_transform", False) + self.with_learnable_separator = kwargs.get("with_learnable_separator", False) + self.hd_transform_order = kwargs.get("hd_transform_order", "glb_sub") + # with_hd_transform and with_learnable_separator should have same value + assert ( + self.use_hd_transform == self.with_learnable_separator + ), "use_hd_transform and with_learnable_separator should have same value" + if self.with_learnable_separator: + assert self.use_hd_transform, "learnable separator is only for hd transform" + # 1024 * 4, merge spatial to channel dimension + self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4])) + self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4])) + logger.info( + f"learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}" + ) + + projection_cls = kwargs.get("projection_cls", "linear") + if projection_cls == "linear": + self.img_projection = nn.Linear(image_dim_out, hidden_size) + elif projection_cls == "mlp" and self.use_hd_transform: + dim_projection = hidden_size + depth = 2 + layers = [nn.Linear(image_dim_out * 4, dim_projection)] + for _ in range(1, depth): + layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)]) + self.img_projection = nn.Sequential(*layers) + elif projection_cls == "mlp": + dim_projection = hidden_size + depth = 2 + layers = [nn.Linear(image_dim_out, dim_projection)] + for _ in range(1, depth): + layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)]) + self.img_projection = nn.Sequential(*layers) + else: + raise NotImplementedError(f"projection_cls = {projection_cls}, not implemented") + + self.vocab_size = config.vocab_size + self.img_features = None + + if isinstance(config.img_processor, dict): + self.layer_idx = config.img_processor.get("layer_idx", -2) + self.type_feature = config.img_processor.get("type_feature", "patch") + else: + self.layer_idx = -2 + self.type_feature = "patch" + + def set_img_features(self, img_features: torch.FloatTensor) -> None: + self.img_features = img_features + + def set_img_sizes(self, img_sizes: torch.LongTensor) -> None: + self.img_sizes = img_sizes + + def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor: + LAYER_IDX = self.layer_idx + TYPE_FEATURE = self.type_feature + + img_processor_output = self.img_processor(img_embeds, output_hidden_states=True) + img_feature = img_processor_output.hidden_states[LAYER_IDX] + + if TYPE_FEATURE == "patch": + patch_feature = img_feature[:, 1:] + return patch_feature + + raise NotImplementedError + + def forward( + self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, image_sizes=None + ) -> torch.FloatTensor: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + # positions for image tokens + positions = torch.nonzero((input_ids < 0) & (input_ids > -MAX_INPUT_ID), as_tuple=True) + has_image = len(positions[0].tolist()) > 0 + input_ids = input_ids.clamp_min(0).clamp_max(self.vocab_size).detach() + hidden_states = self.wte(input_ids) + + if has_image: + assert self.use_hd_transform + num_images, num_crops, c, h, w = pixel_values.shape + assert c == 3 and h == w == 336 + img_features = self.get_img_features(pixel_values.flatten(0, 1)).reshape( + num_images, num_crops, -1, self.image_dim_out + ) + image_features_proj = self.hd_feature_transform(img_features, image_sizes) + hidden_states = hidden_states.index_put(positions, image_features_proj, accumulate=False) + + if self.drop is not None: + hidden_states = self.drop(hidden_states) + + return hidden_states + + def hd_feature_transform(self, image_features, image_sizes): + """ + image_features: (num_images, num_crops+1, 24*24, 1024) + """ + assert self.hd_transform_order == "sub_glb", f"hd_transform_order `{self.hd_transform_order}` not implemented" + if isinstance(self.img_projection, nn.Sequential): + target_device = self.img_projection[0].bias.device + target_dtype = self.img_projection[0].bias.dtype + else: # It's a single nn.Linear layer + target_device = self.img_projection.bias.device + target_dtype = self.img_projection.bias.dtype + + global_image_features = image_features[:, 0] # (num_images, 24*24, 1024) + # global feature can be viewed as a special HD case with num_crops 1x1 + global_image_features_hd = self.reshape_hd_patches_2x2merge(global_image_features, 1, 1) + global_image_features_hd_newline = self.add_image_newline(global_image_features_hd) + + all_image_embeddings = [] + # need a for loop to process each image because of different image sizes + # (patch arrangement is different for each image) + for i, img_size in enumerate(image_sizes): + h, w = img_size + h_crop = h // 336 + w_crop = w // 336 + num_crops = h_crop * w_crop + + # NOTE: real num_crops is padded + # (num_crops, 24*24, 1024) + sub_image_features = image_features[i, 1 : 1 + num_crops] + sub_image_features_hd = self.reshape_hd_patches_2x2merge(sub_image_features, h_crop, w_crop) + sub_image_features_hd_newline = self.add_image_newline(sub_image_features_hd) + + # [sub features, separator, global features] + all_image_embeddings.extend( + [ + sub_image_features_hd_newline.squeeze(0), # (h_crop*12*(w_crop*12+1), 4096) + self.glb_GN.squeeze(0), + global_image_features_hd_newline[i], + ] + ) + + image_features_proj = self.img_projection( + torch.cat(all_image_embeddings, dim=0).to(target_device).to(target_dtype) + ) + + return image_features_proj + + def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): + """ + image_features: (num_images*num_crops, 24*24, 1024) + output: (num_images, h_crop*12, w_crop*12, 4096), h_crop*w_crop == num_crops + """ + N, L, C = image_features.shape + assert L == 24 * 24 and C == 1024 and N % (h_crop * w_crop) == 0 + num_images = N // (h_crop * w_crop) + H = int(L**0.5) + image_features_hd = ( + image_features.reshape(N, H, H, C) # N, 24, 24, 1024 + .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024 + .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024 + .reshape(N, -1, 4 * C) # N, 144, 4096 + .reshape(num_images, h_crop, w_crop, H // 2, H // 2, -1) # n_img, h_crop, w_crop, 12, 12, 4096 + .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096 + .reshape(num_images, h_crop * H // 2, w_crop * H // 2, 4 * C) # n_img, h_crop*12, w_crop*12, 4096 + ) + + return image_features_hd + + def add_image_newline(self, image_features_hd): + """ + image_features_hd: (num_images, h_crop*12, w_crop*12, 4096) + output: (num_images, (h_crop*12) * (w_crop*12+1), 4096) + """ + num_images, h, w, hid_dim = image_features_hd.shape + # add the newline token to the HD image feature patches + newline_embeddings = self.sub_GN.expand(num_images, h, -1, -1) # (n_img, h, 1, hid_dim) + image_features_hd_newline = torch.cat([image_features_hd, newline_embeddings], dim=2).reshape( + num_images, -1, hid_dim + ) + return image_features_hd_newline + + +class Phi3RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Phi3RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +class Phi3RotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.register_buffer("inv_freq", None, persistent=False) + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if self.inv_freq is None: + self.inv_freq = 1.0 / ( + self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim) + ) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Phi3SuScaledRotaryEmbedding(Phi3RotaryEmbedding): + def __init__(self, dim, config, device=None): + super().__init__(dim, config.max_position_embeddings, config.rope_theta, device) + + self.short_factor = config.rope_scaling["short_factor"] + self.long_factor = config.rope_scaling["long_factor"] + self.original_max_position_embeddings = config.original_max_position_embeddings + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + seq_len = seq_len or torch.max(position_ids) + 1 + if seq_len > self.original_max_position_embeddings: + ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device) + else: + ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device) + + inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim + self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape) + + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + + scale = self.max_position_embeddings / self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = math.sqrt(1 + math.log(scale) / math.log(self.original_max_position_embeddings)) + + cos = emb.cos() * scaling_factor + sin = emb.sin() * scaling_factor + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Phi3YarnScaledRotaryEmbedding(Phi3RotaryEmbedding): + def __init__(self, dim, config, device=None): + super().__init__(dim, config.max_position_embeddings, config.rope_theta, device) + + self.short_factor = config.rope_scaling["short_factor"] + self.long_factor = config.rope_scaling["long_factor"] + self.original_max_position_embeddings = config.original_max_position_embeddings + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + seq_len = torch.max(position_ids) + 1 + if seq_len > self.original_max_position_embeddings: + ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device) + else: + ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device) + + inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim + self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape) + + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + + scale = self.max_position_embeddings / self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = 0.1 * math.log(scale) + 1.0 + + cos = emb.cos() * scaling_factor + sin = emb.sin() * scaling_factor + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class Phi3MLP(nn.Module): + def __init__(self, config): + super().__init__() + + self.config = config + self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) + self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) + + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + up_states = self.gate_up_proj(hidden_states) + + gate, up_states = up_states.chunk(2, dim=-1) + up_states = up_states * self.activation_fn(gate) + + return self.down_proj(up_states) + + +class Phi3Attention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: Phi3VConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.original_max_position_embeddings = config.original_max_position_embeddings + self.rope_theta = config.rope_theta + self.rope_scaling = config.rope_scaling + self.is_causal = True + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + op_size = self.num_heads * self.head_dim + 2 * (self.num_key_value_heads * self.head_dim) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.qkv_proj = nn.Linear(self.hidden_size, op_size, bias=False) + self._init_rope() + + def _init_rope(self): + if self.rope_scaling is None: + self.rotary_emb = Phi3RotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + else: + scaling_type = self.config.rope_scaling["type"] + if scaling_type == "su": + self.rotary_emb = Phi3SuScaledRotaryEmbedding(self.head_dim, self.config) + elif scaling_type == "yarn": + self.rotary_emb = Phi3YarnScaledRotaryEmbedding(self.head_dim, self.config) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + logger.warning_once("You are not running the flash-attention implementation, expect numerical differences.") + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class Phi3FlashAttention2(Phi3Attention): + """ + Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # Phi3FlashAttention2 attention does not support output_attentions + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library." + ) + raise ValueError("The current flash attention version does not support sliding window attention.") + + output_attentions = False + + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + + # overwrite attention_mask with padding_mask + attention_mask = kwargs.pop("padding_mask") + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=rotary_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_dropout = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. + + if query_states.dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.qkv_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=attn_dropout, + use_sliding_windows=use_sliding_windows, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + query_length, + dropout=0.0, + softmax_scale=None, + use_sliding_windows=False, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + use_sliding_windows (`bool`, *optional*): + Whether to activate sliding window attention. + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + if not use_sliding_windows: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + +class Phi3SdpaAttention(Phi3Attention): + """ + Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `Phi3Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from Phi3Attention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "Phi3Model is using Phi3SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + qkv = self.qkv_proj(hidden_states) + query_pos = self.num_heads * self.head_dim + query_states = qkv[..., :query_pos] + key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim] + value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :] + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal=self.is_causal and attention_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +class Phi3DecoderLayer(nn.Module): + def __init__(self, config: Phi3VConfig, layer_idx: int): + super().__init__() + + self.config = config + self.self_attn = PHI3_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) + + self.mlp = Phi3MLP(config) + self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.resid_attn_dropout = nn.Dropout(config.resid_pdrop) + self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop) + self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + """ + Args: + hidden_states (`torch.FloatTensor`): + input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + position_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + attn_outputs, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = residual + self.resid_attn_dropout(attn_outputs) + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + self.resid_mlp_dropout(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +PHI3V_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Phi3VConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.", + PHI3V_START_DOCSTRING, +) +class Phi3VPreTrainedModel(PreTrainedModel): + config_class = Phi3VConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Phi3DecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = False + _supports_cache_class = True + + _version = "0.0.5" + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +PHI3V_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance; + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`]. + See [`Phi3ImageProcessor.__call__`] for details. + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): + The sizes of the images in the batch, being (height, width) for each image. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.", + PHI3V_START_DOCSTRING, +) +class Phi3VModel(Phi3VPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`] + + Args: + config: Phi3VConfig + """ + + def __init__(self, config: Phi3VConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.embed_dropout = nn.Dropout(config.embd_pdrop) + + self.vision_embed_tokens = None + if isinstance(config.embd_layer, dict): + # vision embedding layer + embedding_config = {"embedding_cls": config.embd_layer["embedding_cls"], **config.embd_layer} + self.vision_embed_tokens = Phi3ImageEmbedding(config, wte=self.embed_tokens, **embedding_config) + # # set wte the same for vision embedding + # self.vision_embed_tokens.wte.weight = self.embed_tokens.weight + + self.layers = nn.ModuleList( + [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape[:2] + elif inputs_embeds is not None: + batch_size, seq_length = inputs_embeds.shape[:2] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + past_key_values_length = 0 + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + if use_cache: + use_legacy_cache = not isinstance(past_key_values, Cache) + if use_legacy_cache: + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + past_key_values_length = past_key_values.get_usable_length(seq_length) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + if pixel_values is not None and image_sizes is not None: + assert self.vision_embed_tokens is not None, "Vision embedding layer is not defined" + inputs_embeds = self.vision_embed_tokens(input_ids, pixel_values=pixel_values, image_sizes=image_sizes) + else: + inputs_embeds = self.embed_tokens(input_ids) + + if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache: + is_padding_right = attention_mask[:, -1].sum().item() != batch_size + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'" + " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to " + " call `tokenizer.padding_side = 'left'` before tokenizing the input. " + ) + + if self._attn_implementation == "flash_attention_2": + # 2d mask is passed through the layers + attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None + else: + # 4d mask is passed through the layers + attention_mask = _prepare_4d_causal_attention_mask( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window=self.config.sliding_window, + ) + + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + attention_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class Phi3VForCausalLM(Phi3VPreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = Phi3VModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_sizes: Optional[torch.LongTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, Phi3ForCausalLM + + >>> model = Phi3ForCausalLM.from_pretrained("microsoft/phi-3-mini-4k-instruct") + >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-3-mini-4k-instruct") + + >>> prompt = "This is an example script ." + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + 'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum' + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + image_sizes=image_sizes, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + logits_to_keep=None, + **kwargs, + ): + # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the + # process + + # When the first time input length reached long and short factor switching point, enforce re-compute cache + # It will cause downside of slower at this single token position, however, better than current failure. + if ( + past_key_values + and self.config.rope_scaling + and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 + ): + past_length = cache_position[0] + if past_length <= self.config.original_max_position_embeddings: + past_key_values = None + + model_inputs = super().prepare_inputs_for_generation( + input_ids=input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + use_cache=use_cache, + logits_to_keep=logits_to_keep, + **kwargs, + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +@add_start_docstrings( + """ + The [`Phi3VModel`] with a sequence classification head on top (linear layer). + + [`Phi3VForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + PHI3V_START_DOCSTRING, +) +class Phi3VForSequenceClassification(Phi3VPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = Phi3VModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @can_return_tuple + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> SequenceClassifierOutputWithPast: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + + model_outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + hidden_states = model_outputs.last_hidden_state + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + last_non_pad_token = -1 + elif input_ids is not None: + # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id + non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) + token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32) + last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) + else: + last_non_pad_token = -1 + logger.warning_once( + f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be " + "unexpected if using padding tokens in conjunction with `inputs_embeds.`" + ) + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token] + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=model_outputs.past_key_values, + hidden_states=model_outputs.hidden_states, + attentions=model_outputs.attentions, + ) + + +@add_start_docstrings( + """ + [`Phi3VModel`] with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for + Named-Entity-Recognition (NER) tasks. + """, + PHI3V_START_DOCSTRING, +) +class Phi3VForTokenClassification(Phi3VPreTrainedModel): + def __init__(self, config: Phi3VConfig): + super().__init__(config) + self.num_labels = config.num_labels + + self.model = Phi3VModel(config) + if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None: + classifier_dropout = config.classifier_dropout + elif hasattr(config, "hidden_dropout") and config.hidden_dropout is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + model_outputs = self.model( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = model_outputs[0] + hidden_states = self.dropout(hidden_states) + logits = self.classifier(hidden_states) + + loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(logits.device) + batch_size, seq_length = labels.shape + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length) + ) + + if not return_dict: + output = (logits,) + model_outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=model_outputs.hidden_states, + attentions=model_outputs.attentions, + ) + + +__all__ = [ + "CLIPAttentionFA2", + "Phi3Attention", + "Phi3DecoderLayer", + "Phi3FlashAttention2", + "Phi3ImageEmbedding", + "Phi3MLP", + "Phi3RMSNorm", + "Phi3RotaryEmbedding", + "Phi3SdpaAttention", + "Phi3SuScaledRotaryEmbedding", + "Phi3VForCausalLM", + "Phi3VForSequenceClassification", + "Phi3VForTokenClassification", + "Phi3VModel", + "Phi3VPreTrainedModel", + "Phi3YarnScaledRotaryEmbedding", +] From 60f85e69c287a8e7b63f1e7e34e81a6d89217cf2 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 01:50:44 -0700 Subject: [PATCH 39/48] Minor adjustments to modular and test processor --- src/transformers/models/phi3_v/modeling_phi3_v.py | 11 ----------- src/transformers/models/phi3_v/modular_phi3_v.py | 11 ----------- tests/models/phi3_v/test_processor_phi3_v.py | 1 - 3 files changed, 23 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 8f0962ad5aa8..d0ef923fe28b 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -1760,20 +1760,9 @@ def forward( __all__ = [ - "CLIPAttentionFA2", - "Phi3Attention", - "Phi3DecoderLayer", - "Phi3FlashAttention2", - "Phi3ImageEmbedding", - "Phi3MLP", - "Phi3RMSNorm", - "Phi3RotaryEmbedding", - "Phi3SdpaAttention", - "Phi3SuScaledRotaryEmbedding", "Phi3VForCausalLM", "Phi3VForSequenceClassification", "Phi3VForTokenClassification", "Phi3VModel", "Phi3VPreTrainedModel", - "Phi3YarnScaledRotaryEmbedding", ] diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index e0a159c4a940..b62b6ad965e2 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -1771,20 +1771,9 @@ def forward( __all__ = [ - "CLIPAttentionFA2", - "Phi3Attention", - "Phi3DecoderLayer", - "Phi3FlashAttention2", - "Phi3ImageEmbedding", - "Phi3MLP", - "Phi3RMSNorm", - "Phi3RotaryEmbedding", - "Phi3SdpaAttention", - "Phi3SuScaledRotaryEmbedding", "Phi3VForCausalLM", "Phi3VForSequenceClassification", "Phi3VForTokenClassification", "Phi3VModel", "Phi3VPreTrainedModel", - "Phi3YarnScaledRotaryEmbedding", ] diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index 23f68f8f5896..31c0062cf654 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -17,7 +17,6 @@ import tempfile import unittest from io import BytesIO -from typing import Optional import numpy as np import requests From c2a939bdea8936a8fdcdc8324341dea60c38a652 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 02:53:16 -0700 Subject: [PATCH 40/48] Minor adjustment to modular it's FINE --- src/transformers/models/phi3_v/modular_phi3_v.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index b62b6ad965e2..9870fd263901 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -79,11 +79,6 @@ projection_dim=768, ) -PHI3_ATTENTION_CLASSES = { - "eager": Phi3Attention, - "flash_attention_2": Phi3FlashAttention2, - "sdpa": Phi3SdpaAttention, -} class CLIPAttentionFA2(CLIPAttention): """Add flash attention 2 to CLIPAttention. (This is only used in the vision encoder)""" @@ -1021,6 +1016,13 @@ def forward( return attn_output, None, past_key_value +PHI3_ATTENTION_CLASSES = { + "eager": Phi3Attention, + "flash_attention_2": Phi3FlashAttention2, + "sdpa": Phi3SdpaAttention, +} + + class Phi3DecoderLayer(nn.Module): def __init__(self, config: Phi3VConfig, layer_idx: int): super().__init__() From 72b01400b5ff5c4b350b5a9ee3c19bd2f92d51a2 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 03:01:54 -0700 Subject: [PATCH 41/48] removed vestigial import --- src/transformers/models/phi3_v/modular_phi3_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index 9870fd263901..ef8620c9e90e 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -47,7 +47,7 @@ from ...configuration_utils import PretrainedConfig from ...models.clip.configuration_clip import CLIPVisionConfig from ...models.clip.modeling_clip import CLIPAttention, CLIPVisionModel -from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv, rotate_half +from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv logger = logging.get_logger(__name__) From 5e61ff20fbd3f2c8780b5ee924feb5692b424fb3 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 03:21:01 -0700 Subject: [PATCH 42/48] More import changes --- src/transformers/models/phi3_v/modeling_phi3_v.py | 3 +++ src/transformers/models/phi3_v/modular_phi3_v.py | 13 ++----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index d0ef923fe28b..5a59a7b011e0 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -4,12 +4,15 @@ # the file from the modular. If any change should be done, please apply the change to the # modular_phi3_v.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + import math import warnings from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F +from flash_attn import flash_attn_func, flash_attn_varlen_func +from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from torch import nn from torch.nn import CrossEntropyLoss diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index ef8620c9e90e..904c112d532c 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -1,6 +1,5 @@ """PyTorch Phi-3-V model.""" -import inspect import math import warnings from typing import List, Optional, Tuple, Union @@ -33,16 +32,8 @@ ) from .configuration_phi3_v import Phi3VConfig - -try: - from flash_attn import flash_attn_func, flash_attn_varlen_func - from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - - _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) -except ImportError: - pass - -import torch +from flash_attn import flash_attn_func, flash_attn_varlen_func +from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from ...configuration_utils import PretrainedConfig from ...models.clip.configuration_clip import CLIPVisionConfig From d7f4b4236a48db28496cff41f2ac9a7e0181f612 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 03:40:42 -0700 Subject: [PATCH 43/48] missing variable --- src/transformers/models/phi3_v/modeling_phi3_v.py | 5 ++++- src/transformers/models/phi3_v/modular_phi3_v.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 5a59a7b011e0..1a0e1c2fe1c9 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -4,9 +4,9 @@ # the file from the modular. If any change should be done, please apply the change to the # modular_phi3_v.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 - import math import warnings +from inspect import signature from typing import List, Optional, Tuple, Union import torch @@ -599,6 +599,9 @@ def forward( return attn_output, attn_weights, past_key_value +_flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) + + def _get_unpad_data(attention_mask): seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index 904c112d532c..dd2dcfb43379 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -1,5 +1,5 @@ """PyTorch Phi-3-V model.""" - +from inspect import signature import math import warnings from typing import List, Optional, Tuple, Union @@ -53,6 +53,8 @@ MAX_INPUT_ID = int(1e9) +_flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) + CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( attention_dropout=0.0, dropout=0.0, From ea3ddd1ba5e3883ed9eea3ed1263ac48600b5b21 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 16:57:39 -0700 Subject: [PATCH 44/48] various fixes --- .../models/phi3_v/modeling_phi3_v.py | 1 + .../models/phi3_v/modular_phi3_v.py | 24 ++-- .../models/phi3_v/processing_phi3_v.py | 17 ++- tests/models/phi3_v/test_modeling_phi3_v.py | 113 ------------------ tests/models/phi3_v/test_processor_phi3_v.py | 2 +- 5 files changed, 22 insertions(+), 135 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 1a0e1c2fe1c9..935c29b653c1 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -4,6 +4,7 @@ # the file from the modular. If any change should be done, please apply the change to the # modular_phi3_v.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + import math import warnings from inspect import signature diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index dd2dcfb43379..acbe98e725fa 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -1,17 +1,21 @@ """PyTorch Phi-3-V model.""" -from inspect import signature + import math import warnings +from inspect import signature from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F import torch.utils.checkpoint +from flash_attn import flash_attn_func, flash_attn_varlen_func +from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from torch import nn from torch.nn import CrossEntropyLoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache +from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask from ...modeling_outputs import ( @@ -21,6 +25,9 @@ TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel +from ...models.clip.configuration_clip import CLIPVisionConfig +from ...models.clip.modeling_clip import CLIPAttention, CLIPVisionModel +from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv from ...utils import ( add_code_sample_docstrings, add_start_docstrings, @@ -32,14 +39,6 @@ ) from .configuration_phi3_v import Phi3VConfig -from flash_attn import flash_attn_func, flash_attn_varlen_func -from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa - -from ...configuration_utils import PretrainedConfig -from ...models.clip.configuration_clip import CLIPVisionConfig -from ...models.clip.modeling_clip import CLIPAttention, CLIPVisionModel -from ...models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv - logger = logging.get_logger(__name__) @@ -150,9 +149,9 @@ def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None: self.with_learnable_separator = kwargs.get("with_learnable_separator", False) self.hd_transform_order = kwargs.get("hd_transform_order", "glb_sub") # with_hd_transform and with_learnable_separator should have same value - assert ( - self.use_hd_transform == self.with_learnable_separator - ), "use_hd_transform and with_learnable_separator should have same value" + assert self.use_hd_transform == self.with_learnable_separator, ( + "use_hd_transform and with_learnable_separator should have same value" + ) if self.with_learnable_separator: assert self.use_hd_transform, "learnable separator is only for hd transform" # 1024 * 4, merge spatial to channel dimension @@ -920,6 +919,7 @@ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query (max_seqlen_in_batch_q, max_seqlen_in_batch_k), ) + class Phi3SdpaAttention(Phi3Attention): """ Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from diff --git a/src/transformers/models/phi3_v/processing_phi3_v.py b/src/transformers/models/phi3_v/processing_phi3_v.py index 57a9fd421c45..c9e5395a612f 100644 --- a/src/transformers/models/phi3_v/processing_phi3_v.py +++ b/src/transformers/models/phi3_v/processing_phi3_v.py @@ -43,9 +43,8 @@ class Phi3VProcessor(ProcessorMixin): The image processor is a required input. tokenizer ([`LlamaTokenizerFast`]): The tokenizer is a required input. - chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string. - chat_template (`str`, *optional*): """ attributes = ["image_processor", "tokenizer"] @@ -57,7 +56,7 @@ def __init__(self, image_processor, tokenizer, chat_template: str = None, **kwar self.image_processor = image_processor self.tokenizer = tokenizer self.num_img_tokens = image_processor.num_img_tokens - self.img_tokens = [f"<|image_{i+1}|>" for i in range(1000000)] + self.img_tokens = [f"<|image_{i + 1}|>" for i in range(1000000)] super().__init__(image_processor, tokenizer, chat_template=chat_template, **kwargs) def __call__( @@ -190,13 +189,13 @@ def _convert_images_texts_to_inputs( unique_image_ids = sorted(set(image_ids)) # image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be [1, 4, 5] # check the condition - assert unique_image_ids == list( - range(1, len(unique_image_ids) + 1) - ), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}" + assert unique_image_ids == list(range(1, len(unique_image_ids) + 1)), ( + f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}" + ) # total images must be the same as the number of image tags - assert ( - len(unique_image_ids) == len(images) - ), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images" + assert len(unique_image_ids) == len(images), ( + f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images" + ) image_ids_pad = [[-iid] * num_img_tokens[iid - 1] for iid in image_ids] diff --git a/tests/models/phi3_v/test_modeling_phi3_v.py b/tests/models/phi3_v/test_modeling_phi3_v.py index 39389e0b6901..02f91cc5294c 100644 --- a/tests/models/phi3_v/test_modeling_phi3_v.py +++ b/tests/models/phi3_v/test_modeling_phi3_v.py @@ -169,119 +169,6 @@ def create_and_check_model( result = model(input_ids) self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) - # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_model_as_decoder with Llama->Phi3V - def create_and_check_model_as_decoder( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - encoder_hidden_states, - encoder_attention_mask, - ): - config.add_cross_attention = True - model = Phi3VModel(config) - model.to(torch_device) - model.eval() - result = model( - input_ids, - attention_mask=input_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - ) - result = model( - input_ids, - attention_mask=input_mask, - encoder_hidden_states=encoder_hidden_states, - ) - result = model(input_ids, attention_mask=input_mask) - self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) - - # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_for_causal_lm with Llama->Phi3V - def create_and_check_for_causal_lm( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - encoder_hidden_states, - encoder_attention_mask, - ): - model = Phi3VForCausalLM(config=config) - model.to(torch_device) - model.eval() - result = model(input_ids, attention_mask=input_mask, labels=token_labels) - self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) - - # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.create_and_check_decoder_model_past_large_inputs with Llama->Phi3V - def create_and_check_decoder_model_past_large_inputs( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - encoder_hidden_states, - encoder_attention_mask, - ): - config.is_decoder = True - config.add_cross_attention = True - model = Phi3VForCausalLM(config=config) - model.to(torch_device) - model.eval() - - # first forward pass - outputs = model( - input_ids, - attention_mask=input_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - use_cache=True, - ) - past_key_values = outputs.past_key_values - - # create hypothetical multiple next token and extent to next_input_ids - next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) - next_mask = ids_tensor((self.batch_size, 3), vocab_size=2) - - # append to next input_ids and - next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) - next_attention_mask = torch.cat([input_mask, next_mask], dim=-1) - - output_from_no_past = model( - next_input_ids, - attention_mask=next_attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - output_hidden_states=True, - )["hidden_states"][0] - output_from_past = model( - next_tokens, - attention_mask=next_attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - past_key_values=past_key_values, - output_hidden_states=True, - )["hidden_states"][0] - - # select random slice - random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() - output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach() - output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() - - self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) - - # test that outputs are equal for slice - self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) - # Copied from tests.models.llama.test_modeling_llama.LlamaModelTester.prepare_config_and_inputs_for_common def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() diff --git a/tests/models/phi3_v/test_processor_phi3_v.py b/tests/models/phi3_v/test_processor_phi3_v.py index 31c0062cf654..738d9c84bb56 100644 --- a/tests/models/phi3_v/test_processor_phi3_v.py +++ b/tests/models/phi3_v/test_processor_phi3_v.py @@ -106,7 +106,7 @@ def test_image_token_count(self): text_str = "In this image, we see\n" text = text_str + image_str - inputs = processor(text=text, images=self.image1, return_tensors='pt') + inputs = processor(text=text, images=self.image1, return_tensors="pt") tokenized_sentence = processor.tokenizer(text_str) split_image1_tokens = self.get_expected_input_ids(processor, self.image1) expected_input_ids = [tokenized_sentence["input_ids"] + split_image1_tokens] From 96a3d776f30829f1ba626815545545242f51e932 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Sun, 6 Apr 2025 17:56:31 -0700 Subject: [PATCH 45/48] import adjustments --- src/transformers/models/phi3_v/modeling_phi3_v.py | 7 +++++-- src/transformers/models/phi3_v/modular_phi3_v.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 935c29b653c1..0435ef8d6f77 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -12,8 +12,6 @@ import torch import torch.nn.functional as F -from flash_attn import flash_attn_func, flash_attn_varlen_func -from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from torch import nn from torch.nn import CrossEntropyLoss @@ -37,6 +35,7 @@ add_start_docstrings, add_start_docstrings_to_model_forward, can_return_tuple, + is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10, logging, replace_return_docstrings, @@ -44,6 +43,10 @@ from .configuration_phi3_v import Phi3VConfig +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input + logger = logging.get_logger(__name__) _CHECKPOINT_FOR_DOC = "microsoft/Phi-3.5-vision-instruct" diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index acbe98e725fa..8e29b091d03b 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -8,8 +8,6 @@ import torch import torch.nn.functional as F import torch.utils.checkpoint -from flash_attn import flash_attn_func, flash_attn_varlen_func -from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa from torch import nn from torch.nn import CrossEntropyLoss @@ -33,6 +31,7 @@ add_start_docstrings, add_start_docstrings_to_model_forward, can_return_tuple, + is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10, logging, replace_return_docstrings, @@ -40,6 +39,10 @@ from .configuration_phi3_v import Phi3VConfig +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input + logger = logging.get_logger(__name__) _CHECKPOINT_FOR_DOC = "microsoft/Phi-3.5-vision-instruct" From 54e7c9ac70b2661561dc7b604f7f642c13a1d1e3 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Apr 2025 14:31:11 +0100 Subject: [PATCH 46/48] _flash_supports_window_size only in flashattn contexts --- src/transformers/models/phi3_v/modular_phi3_v.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index 8e29b091d03b..1b459043dcfd 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -43,6 +43,8 @@ from flash_attn import flash_attn_func, flash_attn_varlen_func from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input + _flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) + logger = logging.get_logger(__name__) _CHECKPOINT_FOR_DOC = "microsoft/Phi-3.5-vision-instruct" @@ -55,7 +57,6 @@ MAX_INPUT_ID = int(1e9) -_flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( attention_dropout=0.0, From c78a141c8c4741f67459cd87f8c12dd927219621 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Apr 2025 15:26:35 +0100 Subject: [PATCH 47/48] Move things around to stop annoying modular --- src/transformers/models/phi3_v/modeling_phi3_v.py | 12 +++++++----- src/transformers/models/phi3_v/modular_phi3_v.py | 8 +++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/phi3_v/modeling_phi3_v.py b/src/transformers/models/phi3_v/modeling_phi3_v.py index 0435ef8d6f77..e01126c1b9c1 100644 --- a/src/transformers/models/phi3_v/modeling_phi3_v.py +++ b/src/transformers/models/phi3_v/modeling_phi3_v.py @@ -46,6 +46,9 @@ if is_flash_attn_2_available(): from flash_attn import flash_attn_func, flash_attn_varlen_func from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input +else: + flash_attn_func = None + logger = logging.get_logger(__name__) @@ -89,6 +92,7 @@ def forward( MAX_INPUT_ID = int(1e9) + CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( attention_dropout=0.0, dropout=0.0, @@ -603,9 +607,6 @@ def forward( return attn_output, attn_weights, past_key_value -_flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) - - def _get_unpad_data(attention_mask): seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() @@ -632,6 +633,7 @@ def __init__(self, *args, **kwargs): # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + self._flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) def forward( self, @@ -645,7 +647,7 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: # Phi3FlashAttention2 attention does not support output_attentions - if not _flash_supports_window_size: + if not self._flash_supports_window_size: logger.warning_once( "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library." ) @@ -693,7 +695,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) use_sliding_windows = ( - _flash_supports_window_size + self._flash_supports_window_size and getattr(self.config, "sliding_window", None) is not None and kv_seq_len > self.config.sliding_window ) diff --git a/src/transformers/models/phi3_v/modular_phi3_v.py b/src/transformers/models/phi3_v/modular_phi3_v.py index 1b459043dcfd..1f2043806f90 100644 --- a/src/transformers/models/phi3_v/modular_phi3_v.py +++ b/src/transformers/models/phi3_v/modular_phi3_v.py @@ -42,8 +42,9 @@ if is_flash_attn_2_available(): from flash_attn import flash_attn_func, flash_attn_varlen_func from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input +else: + flash_attn_func = None - _flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) logger = logging.get_logger(__name__) @@ -632,6 +633,7 @@ def __init__(self, *args, **kwargs): # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + self._flash_supports_window_size = "window_size" in list(signature(flash_attn_func).parameters) def forward( self, @@ -645,7 +647,7 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: # Phi3FlashAttention2 attention does not support output_attentions - if not _flash_supports_window_size: + if not self._flash_supports_window_size: logger.warning_once( "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library." ) @@ -693,7 +695,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) use_sliding_windows = ( - _flash_supports_window_size + self._flash_supports_window_size and getattr(self.config, "sliding_window", None) is not None and kv_seq_len > self.config.sliding_window ) From 2cb360b35a9531933e127345a3b036d94b42d727 Mon Sep 17 00:00:00 2001 From: Dahlbomii Date: Wed, 20 Aug 2025 11:52:49 -0700 Subject: [PATCH 48/48] small tuple fix --- docs/source/en/model_doc/phi3_v.md | 2 +- src/transformers/models/auto/image_processing_auto.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/phi3_v.md b/docs/source/en/model_doc/phi3_v.md index efcb09909656..08e851a5be44 100644 --- a/docs/source/en/model_doc/phi3_v.md +++ b/docs/source/en/model_doc/phi3_v.md @@ -1,4 +1,4 @@ -