From dc82ee9acaa4a8246e2967381b9c0e33c59c8609 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Fri, 12 Dec 2025 14:58:19 +0000 Subject: [PATCH 01/10] Fix tied weight keys sam2 video --- src/transformers/models/sam2_video/modeling_sam2_video.py | 5 ----- src/transformers/models/sam2_video/modular_sam2_video.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/transformers/models/sam2_video/modeling_sam2_video.py b/src/transformers/models/sam2_video/modeling_sam2_video.py index d107016ccfc2..f9d85c217bef 100644 --- a/src/transformers/models/sam2_video/modeling_sam2_video.py +++ b/src/transformers/models/sam2_video/modeling_sam2_video.py @@ -1559,11 +1559,6 @@ class Sam2VideoModel(Sam2VideoPreTrainedModel): input_modalities = ("video", "text") _can_record_outputs = {"mask_decoder_attentions": OutputRecorder(Sam2VideoTwoWayAttentionBlock, index=2)} _keys_to_ignore_on_load_unexpected = [] - _tied_weights_keys = { - "prompt_encoder.shared_embedding.positional_embedding": "shared_image_embedding.positional_embedding" - } - # need to be ignored, as it's a buffer and will not be correctly detected as tied weight - _keys_to_ignore_on_load_missing = ["prompt_encoder.shared_embedding.positional_embedding"] def __init__(self, config: Sam2VideoConfig): super().__init__(config) diff --git a/src/transformers/models/sam2_video/modular_sam2_video.py b/src/transformers/models/sam2_video/modular_sam2_video.py index e876ee5bda5b..ea2e5d7cadb1 100644 --- a/src/transformers/models/sam2_video/modular_sam2_video.py +++ b/src/transformers/models/sam2_video/modular_sam2_video.py @@ -1446,11 +1446,6 @@ def get_1d_sine_pe(pos_inds, dim, temperature=10000): @auto_docstring class Sam2VideoModel(Sam2Model): input_modalities = ("video", "text") - _tied_weights_keys = { - "prompt_encoder.shared_embedding.positional_embedding": "shared_image_embedding.positional_embedding" - } - # need to be ignored, as it's a buffer and will not be correctly detected as tied weight - _keys_to_ignore_on_load_missing = ["prompt_encoder.shared_embedding.positional_embedding"] _keys_to_ignore_on_load_unexpected = [] _can_record_outputs = {"mask_decoder_attentions": OutputRecorder(Sam2VideoTwoWayAttentionBlock, index=2)} From 98d3e23609f27578c7c9abd618ba8eef23900a94 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Thu, 29 Jan 2026 20:06:14 +0000 Subject: [PATCH 02/10] add promptable concept segmentation pipeline --- src/transformers/__init__.py | 2 + src/transformers/models/auto/modeling_auto.py | 18 + src/transformers/pipelines/__init__.py | 8 + .../promptable_concept_segmentation.py | 404 ++++++++++++++++++ ...pelines_promptable_concept_segmentation.py | 339 +++++++++++++++ 5 files changed, 771 insertions(+) create mode 100644 src/transformers/pipelines/promptable_concept_segmentation.py create mode 100644 tests/pipelines/test_pipelines_promptable_concept_segmentation.py diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e5677dd872f9..bc07a1d8ca56 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -162,6 +162,7 @@ "TextToAudioPipeline", "TokenClassificationPipeline", "VideoClassificationPipeline", + "PromptableConceptSegmentationPipeline", "VisualQuestionAnsweringPipeline", "ZeroShotAudioClassificationPipeline", "ZeroShotClassificationPipeline", @@ -663,6 +664,7 @@ from .pipelines import PipedPipelineDataFormat as PipedPipelineDataFormat from .pipelines import Pipeline as Pipeline from .pipelines import PipelineDataFormat as PipelineDataFormat + from .pipelines import PromptableConceptSegmentationPipeline as PromptableConceptSegmentationPipeline from .pipelines import QuestionAnsweringPipeline as QuestionAnsweringPipeline from .pipelines import TableQuestionAnsweringPipeline as TableQuestionAnsweringPipeline from .pipelines import TextClassificationPipeline as TextClassificationPipeline diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index fa3b1c979939..4c98c82aa2bb 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -1627,6 +1627,14 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ] ) +MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES = OrderedDict( + [ + # Model for Promptable Concept Segmentation mapping + ("sam3", "Sam3Model"), + # facebook/sam3 checkpoint uses sam3_video config but can be used for single-image inference + ("sam3_video", "Sam3Model"), + ] +) MODEL_FOR_KEYPOINT_DETECTION_MAPPING_NAMES = OrderedDict( [ @@ -1797,6 +1805,10 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): MODEL_FOR_MASK_GENERATION_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_MASK_GENERATION_MAPPING_NAMES) +MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING = _LazyAutoMapping( + CONFIG_MAPPING_NAMES, MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES +) + MODEL_FOR_KEYPOINT_DETECTION_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, MODEL_FOR_KEYPOINT_DETECTION_MAPPING_NAMES ) @@ -1826,6 +1838,10 @@ class AutoModelForMaskGeneration(_BaseAutoModelClass): _model_mapping = MODEL_FOR_MASK_GENERATION_MAPPING +class AutoModelForPromptableConceptSegmentation(_BaseAutoModelClass): + _model_mapping = MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING + + class AutoModelForKeypointDetection(_BaseAutoModelClass): _model_mapping = MODEL_FOR_KEYPOINT_DETECTION_MAPPING @@ -2168,6 +2184,7 @@ class AutoModelForAudioTokenization(_BaseAutoModelClass): "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING", "MODEL_FOR_OBJECT_DETECTION_MAPPING", "MODEL_FOR_PRETRAINING_MAPPING", + "MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING", "MODEL_FOR_QUESTION_ANSWERING_MAPPING", "MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING", @@ -2214,6 +2231,7 @@ class AutoModelForAudioTokenization(_BaseAutoModelClass): "AutoModelForNextSentencePrediction", "AutoModelForObjectDetection", "AutoModelForPreTraining", + "AutoModelForPromptableConceptSegmentation", "AutoModelForQuestionAnswering", "AutoModelForSemanticSegmentation", "AutoModelForSeq2SeqLM", diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py index 57c7a806fdf2..da84b4549804 100755 --- a/src/transformers/pipelines/__init__.py +++ b/src/transformers/pipelines/__init__.py @@ -71,6 +71,7 @@ from .keypoint_matching import KeypointMatchingPipeline from .mask_generation import MaskGenerationPipeline from .object_detection import ObjectDetectionPipeline +from .promptable_concept_segmentation import PromptableConceptSegmentationPipeline from .question_answering import QuestionAnsweringArgumentHandler, QuestionAnsweringPipeline from .table_question_answering import TableQuestionAnsweringArgumentHandler, TableQuestionAnsweringPipeline from .text_classification import TextClassificationPipeline @@ -107,6 +108,7 @@ AutoModelForMaskGeneration, AutoModelForMultimodalLM, AutoModelForObjectDetection, + AutoModelForPromptableConceptSegmentation, AutoModelForQuestionAnswering, AutoModelForSemanticSegmentation, AutoModelForSeq2SeqLM, @@ -298,6 +300,12 @@ "default": {"model": ("magic-leap-community/superglue_outdoor", "f4041f8")}, "type": "image", }, + "promptable-concept-segmentation": { + "impl": PromptableConceptSegmentationPipeline, + "pt": (AutoModelForPromptableConceptSegmentation,) if is_torch_available() else (), + "default": {"model": ("facebook/sam3", "main")}, + "type": "multimodal", + }, "any-to-any": { "impl": AnyToAnyPipeline, "tf": (), diff --git a/src/transformers/pipelines/promptable_concept_segmentation.py b/src/transformers/pipelines/promptable_concept_segmentation.py new file mode 100644 index 000000000000..b52cf44849e1 --- /dev/null +++ b/src/transformers/pipelines/promptable_concept_segmentation.py @@ -0,0 +1,404 @@ +# Copyright 2025 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Union, overload + +from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging, requires_backends +from .base import Pipeline, build_pipeline_init_args + + +if is_vision_available(): + from PIL import Image + + from ..image_utils import load_image, valid_images + +if is_torch_available(): + import torch + + from ..models.auto.modeling_auto import MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES + +logger = logging.get_logger(__name__) + + +@add_end_docstrings(build_pipeline_init_args(has_processor=True)) +class PromptableConceptSegmentationPipeline(Pipeline): + """ + Promptable Concept Segmentation pipeline using `Sam3Model`. This pipeline predicts instance segmentation masks + and bounding boxes for objects when you provide an image and prompts. Prompts can be text descriptions + (e.g., "yellow school bus"), visual box exemplars (positive/negative), or combinations of both. + + Example: + + ```python + >>> from transformers import pipeline + + >>> segmenter = pipeline(model="facebook/sam3", task="promptable-concept-segmentation") + >>> segmenter( + ... "http://images.cocodataset.org/val2017/000000077595.jpg", + ... text="ear", + ... ) + [{'score': 0.87, 'box': {'xmin': 120, 'ymin': 45, 'xmax': 210, 'ymax': 130}, 'mask': tensor([...])}, ...] + + >>> # Using box prompts + >>> segmenter( + ... "http://images.cocodataset.org/val2017/000000136466.jpg", + ... input_boxes=[[[59, 144, 76, 163], [87, 148, 104, 159]]], + ... input_boxes_labels=[[1, 1]], + ... ) + [{'score': 0.92, 'box': {'xmin': 59, 'ymin': 144, 'xmax': 76, 'ymax': 163}, 'mask': tensor([...])}, ...] + + >>> # Combined text and negative box + >>> segmenter( + ... "http://images.cocodataset.org/val2017/000000136466.jpg", + ... text="handle", + ... input_boxes=[[[40, 183, 318, 204]]], + ... input_boxes_labels=[[0]], # 0 = negative (exclude this region) + ... ) + [{'score': 0.85, 'box': {'xmin': 250, 'ymin': 100, 'xmax': 280, 'ymax': 150}, 'mask': tensor([...])}, ...] + ``` + + Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) + + This promptable concept segmentation pipeline can currently be loaded from [`pipeline`] using the following task + identifier: `"promptable-concept-segmentation"`. + + See the list of available models on + [huggingface.co/models](https://huggingface.co/models?filter=promptable-concept-segmentation). + """ + + _load_processor = True + _load_image_processor = False + _load_feature_extractor = False + _load_tokenizer = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + requires_backends(self, "vision") + self.check_model_type(MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES) + + # Ensure we have Sam3Processor (not Sam3VideoProcessor) for text and box prompt support + # facebook/sam3 checkpoint loads Sam3VideoProcessor by default, but this pipeline needs Sam3Processor + if self.processor is not None and self.processor.__class__.__name__ == "Sam3VideoProcessor": + from ..models.sam3 import Sam3Processor + + # Try to get the model checkpoint name + model_name = getattr(self.model, "name_or_path", None) + if not model_name and hasattr(self.model, "config"): + model_name = getattr(self.model.config, "_name_or_path", None) + + # Default to facebook/sam3 if we can't determine the model name + # (facebook/sam3 is the canonical checkpoint for this task) + if not model_name: + model_name = "facebook/sam3" + + logger.info( + "Detected Sam3VideoProcessor but promptable-concept-segmentation requires Sam3Processor. " + f"Loading Sam3Processor from {model_name}." + ) + self.processor = Sam3Processor.from_pretrained(model_name) + + @overload + def __call__( + self, + image: Union[str, "Image.Image"], + text: str | None = None, + input_boxes: list[list[list[float]]] | None = None, + input_boxes_labels: list[list[int]] | None = None, + **kwargs: Any, + ) -> list[dict[str, Any]]: ... + + @overload + def __call__(self, image: list[dict[str, Any]], **kwargs: Any) -> list[list[dict[str, Any]]]: ... + + def __call__( + self, + image: Union[str, "Image.Image", list[dict[str, Any]]], + text: str | list[str] | None = None, + input_boxes: list[list[list[float]]] | None = None, + input_boxes_labels: list[list[int]] | None = None, + **kwargs: Any, + ) -> list[dict[str, Any]] | list[list[dict[str, Any]]]: + """ + Segment objects in the image(s) based on the provided prompts. + + Args: + image (`str`, `PIL.Image`, or `list[dict[str, Any]]`): + The pipeline handles three types of images: + + - A string containing an http url pointing to an image + - A string containing a local path to an image + - An image loaded in PIL directly + + You can use this parameter to send directly a list of images, or a dataset or a generator like so: + + ```python + >>> from transformers import pipeline + + >>> segmenter = pipeline(model="facebook/sam3", task="promptable-concept-segmentation") + >>> segmenter( + ... [ + ... { + ... "image": "http://images.cocodataset.org/val2017/000000077595.jpg", + ... "text": "ear", + ... }, + ... { + ... "image": "http://images.cocodataset.org/val2017/000000136466.jpg", + ... "text": "dial", + ... }, + ... ] + ... ) + [[{'score': 0.87, 'box': {...}, 'mask': ...}], [{'score': 0.92, 'box': {...}, 'mask': ...}]] + ``` + + text (`str` or `list[str]`, *optional*): + Text prompt(s) describing the concept to segment (e.g., "yellow school bus", "ear", "handle"). + Can be a single string or a list of strings for batched inference. + + input_boxes (`list[list[list[float]]]`, *optional*): + Visual box prompts in xyxy format [x1, y1, x2, y2] in pixel coordinates. + Structure: [batch, num_boxes, 4]. Used to provide visual exemplars of the concept. + + input_boxes_labels (`list[list[int]]`, *optional*): + Labels for the box prompts. 1 = positive (include), 0 = negative (exclude). + Structure: [batch, num_boxes]. Must match the structure of `input_boxes`. + + threshold (`float`, *optional*, defaults to 0.3): + The probability necessary to make a prediction. + + mask_threshold (`float`, *optional*, defaults to 0.5): + Threshold for binarizing the predicted masks. + + top_k (`int`, *optional*, defaults to None): + The number of top predictions that will be returned by the pipeline. If the provided number is `None` + or higher than the number of predictions available, it will default to the number of predictions. + + timeout (`float`, *optional*, defaults to None): + The maximum time in seconds to wait for fetching images from the web. If None, no timeout is set and + the call may block forever. + + Return: + A list of lists containing prediction results, one list per input image. Each list contains dictionaries + with the following keys: + + - **score** (`float`) -- Confidence score for the detected instance. + - **box** (`dict[str, int]`) -- Bounding box of the detected object in image's original size with keys + `xmin`, `ymin`, `xmax`, `ymax`. + - **mask** (`torch.Tensor`) -- Binary segmentation mask for the instance, shape (height, width). + """ + # Handle different input formats + if isinstance(image, (str, Image.Image)): + inputs = { + "image": image, + "text": text, + "input_boxes": input_boxes, + "input_boxes_labels": input_boxes_labels, + } + elif isinstance(image, (list, tuple)) and valid_images(image): + # Batch of images - create individual inputs for each image + batch_inputs = self._prepare_batch_inputs(image, text, input_boxes, input_boxes_labels) + return list(super().__call__(batch_inputs, **kwargs)) + else: + """ + Supports the following format + - {"image": image, "text": text} + - [{"image": image, "text": text}] + - Generator and datasets + """ + inputs = image + + results = super().__call__(inputs, **kwargs) + return results + + def _prepare_batch_inputs(self, images, text, input_boxes, input_boxes_labels): + """Helper method to prepare batch inputs from separate parameters.""" + # Expand single values to match batch size + num_images = len(images) + text_list = text if isinstance(text, list) else [text] * num_images + boxes_list = input_boxes if input_boxes is not None else [None] * num_images + labels_list = input_boxes_labels if input_boxes_labels is not None else [None] * num_images + + # Create input dict for each image + return ( + { + "image": img, + "text": txt, + "input_boxes": boxes, + "input_boxes_labels": box_labels, + } + for img, txt, boxes, box_labels in zip(images, text_list, boxes_list, labels_list) + ) + + def _sanitize_parameters(self, **kwargs): + preprocess_params = {} + if "timeout" in kwargs: + preprocess_params["timeout"] = kwargs["timeout"] + + postprocess_params = {} + if "threshold" in kwargs: + postprocess_params["threshold"] = kwargs["threshold"] + if "mask_threshold" in kwargs: + postprocess_params["mask_threshold"] = kwargs["mask_threshold"] + if "top_k" in kwargs: + postprocess_params["top_k"] = kwargs["top_k"] + + return preprocess_params, {}, postprocess_params + + def _normalize_boxes_format(self, input_boxes): + """Ensure input_boxes is in the correct format: [batch, num_boxes, 4].""" + if input_boxes is None: + return None + if not isinstance(input_boxes, list): + return [[input_boxes]] + if len(input_boxes) > 0 and not isinstance(input_boxes[0], list): + return [input_boxes] + return input_boxes + + def _normalize_labels_format(self, input_boxes_labels): + """Ensure input_boxes_labels is in the correct format: [batch, num_boxes].""" + if input_boxes_labels is None: + return None + if not isinstance(input_boxes_labels, list): + return [[input_boxes_labels]] + if len(input_boxes_labels) > 0 and not isinstance(input_boxes_labels[0], list): + return [input_boxes_labels] + return input_boxes_labels + + def preprocess(self, inputs, timeout=None): + """ + Preprocess inputs for the model. + + Args: + inputs: Dictionary containing 'image' and optionally 'text', 'input_boxes', 'input_boxes_labels' + timeout: Timeout for image loading + + Returns: + Dictionary with preprocessed model inputs + """ + image = load_image(inputs["image"], timeout=timeout) + text = inputs.get("text") + input_boxes = inputs.get("input_boxes") + input_boxes_labels = inputs.get("input_boxes_labels") + + # Validate that at least one prompt type is provided + if text is None and input_boxes is None: + raise ValueError( + "You must provide at least one prompt type: either 'text' or 'input_boxes'. " + "For example: text='cat' or input_boxes=[[[100, 150, 200, 250]]]" + ) + + # Normalize box formats + input_boxes = self._normalize_boxes_format(input_boxes) + input_boxes_labels = self._normalize_labels_format(input_boxes_labels) + + # Process inputs - pass text, input_boxes, input_boxes_labels as explicit parameters + model_inputs = self.processor( + images=image, + text=text, + input_boxes=input_boxes, + input_boxes_labels=input_boxes_labels, + return_tensors="pt", + ) + model_inputs = model_inputs.to(self.dtype) + + # Store original size for post-processing + target_size = torch.tensor([[image.height, image.width]], dtype=torch.int32) + model_inputs["target_size"] = target_size + + # Store the text prompt for output labeling + model_inputs["prompt_text"] = text + + return model_inputs + + def _forward(self, model_inputs): + """ + Forward pass through the model. + + Args: + model_inputs: Preprocessed model inputs + + Returns: + Model outputs with additional metadata + """ + target_size = model_inputs.pop("target_size") + prompt_text = model_inputs.pop("prompt_text") + + outputs = self.model(**model_inputs) + + return { + "outputs": outputs, + "target_size": target_size, + "prompt_text": prompt_text, + } + + def postprocess(self, model_outputs, threshold=0.3, mask_threshold=0.5, top_k=None): + """ + Post-process model outputs into final predictions. + + Args: + model_outputs: Raw model outputs + threshold: Score threshold for filtering predictions + mask_threshold: Threshold for binarizing masks + top_k: Maximum number of predictions to return + + Returns: + List of dictionaries with 'score', 'box', and 'mask' keys + """ + outputs = model_outputs["outputs"] + target_sizes = model_outputs["target_size"] + prompt_text = model_outputs["prompt_text"] + + # Use processor's post-processing method + results = self.processor.post_process_instance_segmentation( + outputs, + threshold=threshold, + mask_threshold=mask_threshold, + target_sizes=target_sizes.tolist(), + )[0] # Get first batch element + + # Convert to expected output format + final_results = [] + if len(results["scores"]) > 0: + for i in range(len(results["scores"])): + score = results["scores"][i].item() + box_tensor = results["boxes"][i] + mask_tensor = results["masks"][i] + + # Convert box to dict format + xmin, ymin, xmax, ymax = box_tensor.int().tolist() + box_dict = { + "xmin": xmin, + "ymin": ymin, + "xmax": xmax, + "ymax": ymax, + } + + result = { + "score": score, + "box": box_dict, + "mask": mask_tensor, + } + + # Optionally add label if text prompt was provided + if prompt_text is not None: + result["label"] = prompt_text + + final_results.append(result) + + # Sort results by score in descending order + final_results = sorted(final_results, key=lambda x: x["score"], reverse=True) + + # Apply top_k filtering + if top_k is not None and len(final_results) > top_k: + final_results = final_results[:top_k] + + return final_results diff --git a/tests/pipelines/test_pipelines_promptable_concept_segmentation.py b/tests/pipelines/test_pipelines_promptable_concept_segmentation.py new file mode 100644 index 000000000000..c6834c38da81 --- /dev/null +++ b/tests/pipelines/test_pipelines_promptable_concept_segmentation.py @@ -0,0 +1,339 @@ +# Copyright 2025 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from transformers import ( + PromptableConceptSegmentationPipeline, + is_torch_available, + is_vision_available, + pipeline, +) +from transformers.testing_utils import ( + is_pipeline_test, + require_torch, + require_vision, + slow, +) + + +if is_vision_available(): + from PIL import Image +else: + + class Image: + @staticmethod + def open(*args, **kwargs): + pass + + +if is_torch_available(): + import torch + + +@is_pipeline_test +@require_vision +@require_torch +class PromptableConceptSegmentationPipelineTests(unittest.TestCase): + def get_test_pipeline( + self, + model, + tokenizer=None, + image_processor=None, + feature_extractor=None, + processor=None, + dtype="float32", + ): + segmenter = PromptableConceptSegmentationPipeline( + model=model, + processor=processor, + tokenizer=tokenizer, + image_processor=image_processor, + dtype=dtype, + ) + + examples = [ + { + "image": "./tests/fixtures/tests_samples/COCO/000000039769.png", + "text": "cat", + } + ] + return segmenter, examples + + def run_pipeline_test(self, segmenter, examples): + outputs = segmenter(examples[0].get("image"), text=examples[0].get("text"), threshold=0.0) + + n = len(outputs) + self.assertGreater(n, 0) + + # Check output structure + for output in outputs: + self.assertIn("score", output) + self.assertIn("box", output) + self.assertIn("mask", output) + self.assertIsInstance(output["score"], float) + self.assertIsInstance(output["box"], dict) + self.assertIn("xmin", output["box"]) + self.assertIn("ymin", output["box"]) + self.assertIn("xmax", output["box"]) + self.assertIn("ymax", output["box"]) + self.assertTrue(is_torch_available() and isinstance(output["mask"], torch.Tensor)) + + @require_torch + @slow + def test_small_model_pt_text_prompt(self): + """Test pipeline with text-only prompt.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.1, + ) + + # Check that we got results + self.assertGreater(len(outputs), 0) + + # Check structure of first result + result = outputs[0] + self.assertIn("score", result) + self.assertIn("box", result) + self.assertIn("mask", result) + self.assertIn("label", result) + self.assertEqual(result["label"], "cat") + + # Check box format + self.assertIsInstance(result["box"]["xmin"], int) + self.assertIsInstance(result["box"]["ymin"], int) + self.assertIsInstance(result["box"]["xmax"], int) + self.assertIsInstance(result["box"]["ymax"], int) + + # Check mask shape + self.assertEqual(len(result["mask"].shape), 2) # Should be 2D (H, W) + + @require_torch + @slow + def test_small_model_pt_box_prompt(self): + """Test pipeline with box-only prompt.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Use a bounding box around a cat in the image + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + input_boxes=[[[100, 50, 400, 350]]], + input_boxes_labels=[[1]], + threshold=0.1, + ) + + # Check that we got results + self.assertGreater(len(outputs), 0) + + # Check structure + result = outputs[0] + self.assertIn("score", result) + self.assertIn("box", result) + self.assertIn("mask", result) + + @require_torch + @slow + def test_small_model_pt_combined_prompt(self): + """Test pipeline with combined text and box prompts.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Text prompt with a negative box + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + input_boxes=[[[50, 50, 150, 150]]], # Negative box + input_boxes_labels=[[0]], # 0 = negative + threshold=0.1, + ) + + # Should still get results, but filtered by negative box + self.assertGreaterEqual(len(outputs), 0) + + @require_torch + @slow + def test_batched_text_prompts(self): + """Test batching with multiple images and text prompts.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + outputs = segmenter( + [ + { + "image": "./tests/fixtures/tests_samples/COCO/000000039769.png", + "text": "cat", + }, + { + "image": "./tests/fixtures/tests_samples/COCO/000000039769.png", + "text": "remote", + }, + ], + threshold=0.1, + ) + + # Should get a list of lists + self.assertEqual(len(outputs), 2) + self.assertIsInstance(outputs[0], list) + self.assertIsInstance(outputs[1], list) + + @require_torch + @slow + def test_threshold(self): + """Test score threshold filtering.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Get results with low threshold + outputs_low = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.01, + ) + + # Get results with high threshold + outputs_high = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.5, + ) + + # High threshold should give fewer or equal results + self.assertLessEqual(len(outputs_high), len(outputs_low)) + + @require_torch + @slow + def test_mask_threshold(self): + """Test mask binarization threshold.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.1, + mask_threshold=0.5, + ) + + # Check that masks are binary + if len(outputs) > 0: + mask = outputs[0]["mask"] + unique_values = torch.unique(mask) + # Mask should be binary (0 and 1) or close to it + self.assertTrue(all(val in [0, 1, 0.0, 1.0] or (val >= 0 and val <= 1) for val in unique_values)) + + @require_torch + @slow + def test_top_k(self): + """Test top_k parameter to limit number of results.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Get all results + outputs_all = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.01, + ) + + # Get only top 2 + outputs_top2 = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.01, + top_k=2, + ) + + # Should have at most 2 results + self.assertLessEqual(len(outputs_top2), 2) + self.assertLessEqual(len(outputs_top2), len(outputs_all)) + + @require_torch + @slow + def test_dict_input_format(self): + """Test dict input format.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Dict format + outputs = segmenter( + {"image": "./tests/fixtures/tests_samples/COCO/000000039769.png", "text": "cat"}, + threshold=0.1, + ) + + self.assertGreater(len(outputs), 0) + + @require_torch + @slow + def test_no_prompt_error(self): + """Test that error is raised when no prompts are provided.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + with self.assertRaises(ValueError) as context: + segmenter("./tests/fixtures/tests_samples/COCO/000000039769.png") + + self.assertIn("at least one prompt", str(context.exception).lower()) + + @require_torch + @slow + def test_multiple_boxes(self): + """Test with multiple positive boxes.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Multiple positive boxes + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + input_boxes=[[[100, 50, 300, 250], [350, 100, 550, 350]]], + input_boxes_labels=[[1, 1]], + threshold=0.1, + ) + + # Should get results + self.assertGreaterEqual(len(outputs), 0) + + @require_torch + @slow + def test_scores_are_sorted(self): + """Test that results are sorted by score in descending order.""" + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.01, + ) + + if len(outputs) > 1: + scores = [output["score"] for output in outputs] + # Check that scores are sorted in descending order + self.assertEqual(scores, sorted(scores, reverse=True)) + + @require_torch + @slow + def test_automatic_model_processor_conversion(self): + """Test that the pipeline automatically converts Sam3VideoModel/Processor to Sam3Model/Processor.""" + # This should work even though facebook/sam3 has Sam3VideoModel by default + segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + + # Verify correct types were loaded + self.assertEqual(segmenter.model.__class__.__name__, "Sam3Model") + self.assertEqual(segmenter.processor.__class__.__name__, "Sam3Processor") + + # Verify it works functionally + outputs = segmenter( + "./tests/fixtures/tests_samples/COCO/000000039769.png", + text="cat", + threshold=0.3, + ) + + self.assertGreater(len(outputs), 0) + self.assertIn("score", outputs[0]) + self.assertIn("box", outputs[0]) + self.assertIn("mask", outputs[0]) From bdae3799168f6d9d507aa46bf64c644eaea69948 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Fri, 30 Jan 2026 18:27:59 +0000 Subject: [PATCH 03/10] add docs to sam3 --- docs/source/en/model_doc/sam3.md | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/source/en/model_doc/sam3.md b/docs/source/en/model_doc/sam3.md index bae0fe97bde3..8449aaa48560 100644 --- a/docs/source/en/model_doc/sam3.md +++ b/docs/source/en/model_doc/sam3.md @@ -39,6 +39,58 @@ This model was contributed by [yonigozlan](https://huggingface.co/yonigozlan) an ## Usage examples with 🤗 Transformers +### Using the Pipeline + +The simplest way to use SAM3 is through the `promptable-concept-segmentation` pipeline: + +```python +>>> from transformers import pipeline +>>> from PIL import Image +>>> import requests + +>>> # Create pipeline +>>> segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") + +>>> # Load image +>>> image_url = "http://images.cocodataset.org/val2017/000000077595.jpg" +>>> image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB") + +>>> # Segment using text prompt +>>> results = segmenter(image, text="ear", threshold=0.5, mask_threshold=0.5) + +>>> print(f"Found {len(results)} objects") +>>> # Results contain: +>>> # - score: Confidence score for each instance +>>> # - label: The text prompt used +>>> # - box: Bounding box in xyxy format (absolute pixel coordinates) +>>> # - mask: Binary segmentation mask (resized to original image size) + +>>> # You can also use bounding box prompts +>>> # Box in xyxy format: [x1, y1, x2, y2] in pixel coordinates +>>> kitchen_url = "http://images.cocodataset.org/val2017/000000136466.jpg" +>>> kitchen_image = Image.open(requests.get(kitchen_url, stream=True).raw).convert("RGB") + +>>> box_xyxy = [59, 144, 76, 163] +>>> input_boxes = [[box_xyxy]] # [batch, num_boxes, 4] +>>> input_boxes_labels = [[1]] # 1 = positive box + +>>> results = segmenter( +... kitchen_image, +... input_boxes=input_boxes, +... input_boxes_labels=input_boxes_labels, +... threshold=0.5, +... mask_threshold=0.5 +... ) + +>>> print(f"Found {len(results)} objects matching the visual concept") +``` + + + +**Note:** The pipeline output format differs from using the model and processor manually. The pipeline returns a standardized format (list of dicts with `score`, `label`, `box`, `mask`) to ensure consistency across all transformers pipelines, while the processor's `post_process_instance_segmentation()` returns a dict with `scores`, `boxes`, and `masks` as separate tensors. + + + ### Text-Only Prompts ```python From 552bc2b33628c2a4bdd2f711ea098a44d916e29e Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Fri, 30 Jan 2026 19:19:50 +0000 Subject: [PATCH 04/10] add docs and pipeline_model_mapping --- .../tasks/promptable_concept_segmentation.md | 302 ++++++++++++++++++ src/transformers/__init__.py | 2 +- src/transformers/models/auto/modeling_auto.py | 4 +- .../modeling_pp_doclayout_v3.py | 8 +- src/transformers/pipelines/__init__.py | 2 + .../promptable_concept_segmentation.py | 27 +- tests/models/sam3/test_modeling_sam3.py | 4 +- utils/check_docstrings.py | 1 + 8 files changed, 328 insertions(+), 22 deletions(-) create mode 100644 docs/source/en/tasks/promptable_concept_segmentation.md diff --git a/docs/source/en/tasks/promptable_concept_segmentation.md b/docs/source/en/tasks/promptable_concept_segmentation.md new file mode 100644 index 000000000000..8985ac92a0d4 --- /dev/null +++ b/docs/source/en/tasks/promptable_concept_segmentation.md @@ -0,0 +1,302 @@ + + +# Promptable Concept Segmentation + +[[open-in-colab]] + +Promptable Concept Segmentation (PCS) is a computer vision task that detects and segments **all instances** of objects matching a given concept in an image. Unlike traditional instance segmentation that is limited to a fixed set of object classes, PCS can segment objects based on: + +- **Text prompts** (e.g., "yellow school bus", "ear", "dial") +- **Visual prompts** (bounding boxes indicating positive or negative examples) +- **Combined prompts** (text + visual cues) + +For each matching object, PCS returns: +- Binary segmentation masks +- Bounding boxes +- Confidence scores + +> [!NOTE] +> Currently, [SAM3](https://huggingface.co/facebook/sam3) is the primary model supporting this task on the Hub. + +In this guide, you will learn how to: + +- Use the pipeline for quick inference +- Segment objects with text prompts +- Segment objects with bounding box prompts +- Combine text and visual prompts for refined segmentation +- Process multiple images in batches + +Before you begin, make sure you have all the necessary libraries installed: + +```bash +pip install -q transformers +``` + +## Promptable Concept Segmentation pipeline + +The simplest way to try out promptable concept segmentation is to use the [`pipeline`]. Instantiate a pipeline from a [checkpoint on the Hugging Face Hub](https://huggingface.co/models?other=sam3): + +```python +>>> from transformers import pipeline + +>>> segmenter = pipeline("promptable-concept-segmentation", model="facebook/sam3") +``` + +Next, choose an image you'd like to segment objects in. Here we'll use an image from the [COCO dataset](https://cocodataset.org/): + +```py +>>> from PIL import Image +>>> import requests + +>>> url = "http://images.cocodataset.org/val2017/000000077595.jpg" +>>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") +>>> image +``` + +
+ Cats on a couch +
+ +### Text-based segmentation + +Pass the image and a text prompt describing the concept you want to segment: + +```py +>>> results = segmenter(image, text="ear", threshold=0.5, mask_threshold=0.5) +>>> results +[{'score': 0.8492, + 'label': 'ear', + 'box': {'xmin': 335, 'ymin': 149, 'xmax': 369, 'ymax': 186}, + 'mask': tensor([[False, False, False, ..., False, False, False], + [False, False, False, ..., False, False, False], + ...])}, + {'score': 0.8415, + 'label': 'ear', + 'box': {'xmin': 194, 'ymin': 152, 'xmax': 227, 'ymax': 190}, + 'mask': tensor([[False, False, False, ..., False, False, False], + ...])}, + ...] +``` + +The results contain all detected instances of the concept: +- `score`: Confidence score (0-1) +- `label`: The text prompt used +- `box`: Bounding box in `{xmin, ymin, xmax, ymax}` format (absolute pixel coordinates) +- `mask`: Binary segmentation mask (same size as original image) + +### Visualizing results + +Let's visualize the segmentation masks: + +```py +>>> import numpy as np +>>> import matplotlib.pyplot as plt +>>> from matplotlib.patches import Rectangle + +>>> fig, ax = plt.subplots(1, 1, figsize=(10, 8)) +>>> ax.imshow(image) + +>>> # Create a colored overlay for all masks +>>> overlay = np.zeros((*image.size[::-1], 4)) +>>> colors = plt.cm.rainbow(np.linspace(0, 1, len(results))) + +>>> for i, result in enumerate(results): +... mask = result["mask"].numpy() +... box = result["box"] +... score = result["score"] +... +... # Add colored mask +... overlay[mask] = [*colors[i][:3], 0.5] +... +... # Draw bounding box +... rect = Rectangle( +... (box["xmin"], box["ymin"]), +... box["xmax"] - box["xmin"], +... box["ymax"] - box["ymin"], +... linewidth=2, +... edgecolor=colors[i], +... facecolor="none", +... ) +... ax.add_patch(rect) +... ax.text(box["xmin"], box["ymin"] - 5, f"{score:.2f}", color="white", fontsize=12, weight="bold") + +>>> ax.imshow(overlay) +>>> ax.axis("off") +>>> plt.tight_layout() +>>> plt.show() +``` + +### Box-based segmentation + +You can also segment objects using bounding boxes as visual prompts. This is useful when you want to segment specific object instances: + +```py +>>> # Load a different image +>>> kitchen_url = "http://images.cocodataset.org/val2017/000000136466.jpg" +>>> kitchen_image = Image.open(requests.get(kitchen_url, stream=True).raw).convert("RGB") + +>>> # Define a bounding box around a dial (xyxy format: [x1, y1, x2, y2]) +>>> box_xyxy = [59, 144, 76, 163] +>>> input_boxes = [[box_xyxy]] # [batch, num_boxes, 4] +>>> input_boxes_labels = [[1]] # 1 = positive box (include objects like this) + +>>> results = segmenter( +... kitchen_image, +... input_boxes=input_boxes, +... input_boxes_labels=input_boxes_labels, +... threshold=0.5, +... mask_threshold=0.5, +... ) + +>>> print(f"Found {len(results)} objects matching the visual concept") +``` + +Box labels can be: +- `1`: Positive (find objects similar to this) +- `0`: Negative (exclude objects like this) + +### Combined text and visual prompts + +For more precise segmentation, combine text prompts with visual examples: + +```py +>>> # Segment "handle" but exclude the oven handle using a negative box +>>> text = "handle" +>>> oven_handle_box = [40, 183, 318, 204] # Box covering oven handle +>>> input_boxes = [[oven_handle_box]] +>>> input_boxes_labels = [[0]] # 0 = negative (exclude this region) + +>>> results = segmenter( +... kitchen_image, +... text=text, +... input_boxes=input_boxes, +... input_boxes_labels=input_boxes_labels, +... threshold=0.5, +... mask_threshold=0.5, +... ) +>>> # This will segment pot handles but exclude the oven handle +``` + +## Manual inference with model and processor + +While the pipeline is convenient, you may want more control over the inference process. Here's how to use the model and processor directly: + +```py +>>> from transformers import Sam3Processor, Sam3Model +>>> import torch + +>>> device = "cuda" if torch.cuda.is_available() else "cpu" +>>> model = Sam3Model.from_pretrained("facebook/sam3").to(device) +>>> processor = Sam3Processor.from_pretrained("facebook/sam3") +``` + +Load an image: + +```py +>>> url = "http://images.cocodataset.org/val2017/000000077595.jpg" +>>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") +``` + +Prepare inputs and run inference: + +```py +>>> inputs = processor(images=image, text="ear", return_tensors="pt").to(device) + +>>> with torch.no_grad(): +... outputs = model(**inputs) + +>>> # Post-process results +>>> results = processor.post_process_instance_segmentation( +... outputs, +... threshold=0.5, +... mask_threshold=0.5, +... target_sizes=inputs.get("original_sizes").tolist(), +... )[0] + +>>> print(f"Found {len(results['masks'])} objects") +>>> # Results contain: +>>> # - masks: List of binary masks (torch.Tensor) +>>> # - boxes: Bounding boxes in xyxy format (torch.Tensor) +>>> # - scores: Confidence scores (torch.Tensor) +``` + +> [!TIP] +> **Pipeline vs Manual Output Format**: The pipeline returns a standardized format (list of dicts with `score`, `label`, `box`, `mask`) for consistency across transformers. The processor's `post_process_instance_segmentation()` returns separate tensors (`scores`, `boxes`, `masks`) for more flexible post-processing. + +## Batch processing + +You can process multiple images efficiently by batching them together: + +```py +>>> cat_url = "http://images.cocodataset.org/val2017/000000077595.jpg" +>>> kitchen_url = "http://images.cocodataset.org/val2017/000000136466.jpg" +>>> images = [ +... Image.open(requests.get(cat_url, stream=True).raw).convert("RGB"), +... Image.open(requests.get(kitchen_url, stream=True).raw).convert("RGB"), +... ] + +>>> # Different text prompt for each image +>>> text_prompts = ["ear", "dial"] + +>>> inputs = processor(images=images, text=text_prompts, return_tensors="pt").to(device) + +>>> with torch.no_grad(): +... outputs = model(**inputs) + +>>> results = processor.post_process_instance_segmentation( +... outputs, +... threshold=0.5, +... mask_threshold=0.5, +... target_sizes=inputs.get("original_sizes").tolist(), +... ) + +>>> for i, result in enumerate(results): +... print(f"Image {i+1}: {len(result['masks'])} objects found with prompt '{text_prompts[i]}'") +``` + +## Efficient multi-prompt inference + +When running multiple prompts on the same image, pre-compute vision embeddings to avoid redundant computation: + +```py +>>> # Pre-process image and compute vision embeddings once +>>> img_inputs = processor(images=image, return_tensors="pt").to(device) +>>> with torch.no_grad(): +... vision_embeds = model.get_vision_features(pixel_values=img_inputs.pixel_values) + +>>> # Run multiple text prompts efficiently +>>> text_prompts = ["ear", "eye", "nose"] +>>> all_results = [] + +>>> for prompt in text_prompts: +... text_inputs = processor(text=prompt, return_tensors="pt").to(device) +... with torch.no_grad(): +... outputs = model(vision_embeds=vision_embeds, **text_inputs) +... +... results = processor.post_process_instance_segmentation( +... outputs, +... threshold=0.5, +... mask_threshold=0.5, +... target_sizes=img_inputs.get("original_sizes").tolist(), +... )[0] +... all_results.append({"prompt": prompt, "results": results}) + +>>> for item in all_results: +... print(f"Prompt '{item['prompt']}': {len(item['results']['masks'])} objects found") +``` + +This approach significantly speeds up inference when testing multiple concepts on the same image! diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index bc07a1d8ca56..2401c60c8e26 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -155,6 +155,7 @@ "PipedPipelineDataFormat", "Pipeline", "PipelineDataFormat", + "PromptableConceptSegmentationPipeline", "QuestionAnsweringPipeline", "TableQuestionAnsweringPipeline", "TextClassificationPipeline", @@ -162,7 +163,6 @@ "TextToAudioPipeline", "TokenClassificationPipeline", "VideoClassificationPipeline", - "PromptableConceptSegmentationPipeline", "VisualQuestionAnsweringPipeline", "ZeroShotAudioClassificationPipeline", "ZeroShotClassificationPipeline", diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 2e06a433287a..62ab7fbc3559 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -1629,11 +1629,11 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ] ) +# Model for Promptable Concept Segmentation mapping +# facebook/sam3 checkpoint uses sam3_video config but can be used for single-image inference MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES = OrderedDict( [ - # Model for Promptable Concept Segmentation mapping ("sam3", "Sam3Model"), - # facebook/sam3 checkpoint uses sam3_video config but can be used for single-image inference ("sam3_video", "Sam3Model"), ] ) diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index 3891a23dfe89..d717c595ab9e 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -1987,15 +1987,15 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): r""" logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): Classification logits (including no-object) for all queries. - order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Order logits of the final layer of the decoder. - out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): - Masks of the final layer of the decoder. pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). You can use [`~PPDocLayoutV3ImageProcessorFast.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits of the final layer of the decoder. + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): + Masks of the final layer of the decoder. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py index da84b4549804..4f3c17840605 100755 --- a/src/transformers/pipelines/__init__.py +++ b/src/transformers/pipelines/__init__.py @@ -446,6 +446,8 @@ def pipeline(task: Literal["mask-generation"], model: str | PreTrainedModel | No @overload def pipeline(task: Literal["object-detection"], model: str | PreTrainedModel | None = None, config: str | PreTrainedConfig | None = None, tokenizer: str | PreTrainedTokenizer | PreTrainedTokenizerFast | None = None, feature_extractor: str | PreTrainedFeatureExtractor | None = None, image_processor: str | BaseImageProcessor | None = None, processor: str | ProcessorMixin | None = None, revision: str | None = None, use_fast: bool = True, token: str | bool | None = None, device: int | str | torch.device | None = None, device_map: str | dict[str, int | str] | None = None, dtype: str | torch.dtype | None = "auto", trust_remote_code: bool | None = None, model_kwargs: dict[str, Any] | None = None, pipeline_class: Any | None = None, **kwargs: Any) -> ObjectDetectionPipeline: ... @overload +def pipeline(task: Literal["promptable-concept-segmentation"], model: str | PreTrainedModel | None = None, config: str | PreTrainedConfig | None = None, tokenizer: str | PreTrainedTokenizer | PreTrainedTokenizerFast | None = None, feature_extractor: str | PreTrainedFeatureExtractor | None = None, image_processor: str | BaseImageProcessor | None = None, processor: str | ProcessorMixin | None = None, revision: str | None = None, use_fast: bool = True, token: str | bool | None = None, device: int | str | torch.device | None = None, device_map: str | dict[str, int | str] | None = None, dtype: str | torch.dtype | None = "auto", trust_remote_code: bool | None = None, model_kwargs: dict[str, Any] | None = None, pipeline_class: Any | None = None, **kwargs: Any) -> PromptableConceptSegmentationPipeline: ... +@overload def pipeline(task: Literal["question-answering"], model: str | PreTrainedModel | None = None, config: str | PreTrainedConfig | None = None, tokenizer: str | PreTrainedTokenizer | PreTrainedTokenizerFast | None = None, feature_extractor: str | PreTrainedFeatureExtractor | None = None, image_processor: str | BaseImageProcessor | None = None, processor: str | ProcessorMixin | None = None, revision: str | None = None, use_fast: bool = True, token: str | bool | None = None, device: int | str | torch.device | None = None, device_map: str | dict[str, int | str] | None = None, dtype: str | torch.dtype | None = "auto", trust_remote_code: bool | None = None, model_kwargs: dict[str, Any] | None = None, pipeline_class: Any | None = None, **kwargs: Any) -> QuestionAnsweringPipeline: ... @overload def pipeline(task: Literal["table-question-answering"], model: str | PreTrainedModel | None = None, config: str | PreTrainedConfig | None = None, tokenizer: str | PreTrainedTokenizer | PreTrainedTokenizerFast | None = None, feature_extractor: str | PreTrainedFeatureExtractor | None = None, image_processor: str | BaseImageProcessor | None = None, processor: str | ProcessorMixin | None = None, revision: str | None = None, use_fast: bool = True, token: str | bool | None = None, device: int | str | torch.device | None = None, device_map: str | dict[str, int | str] | None = None, dtype: str | torch.dtype | None = "auto", trust_remote_code: bool | None = None, model_kwargs: dict[str, Any] | None = None, pipeline_class: Any | None = None, **kwargs: Any) -> TableQuestionAnsweringPipeline: ... diff --git a/src/transformers/pipelines/promptable_concept_segmentation.py b/src/transformers/pipelines/promptable_concept_segmentation.py index b52cf44849e1..647cf536987e 100644 --- a/src/transformers/pipelines/promptable_concept_segmentation.py +++ b/src/transformers/pipelines/promptable_concept_segmentation.py @@ -196,14 +196,14 @@ def __call__( - **mask** (`torch.Tensor`) -- Binary segmentation mask for the instance, shape (height, width). """ # Handle different input formats - if isinstance(image, (str, Image.Image)): + if isinstance(image, str | Image.Image): inputs = { "image": image, "text": text, "input_boxes": input_boxes, "input_boxes_labels": input_boxes_labels, } - elif isinstance(image, (list, tuple)) and valid_images(image): + elif isinstance(image, list | tuple) and valid_images(image): # Batch of images - create individual inputs for each image batch_inputs = self._prepare_batch_inputs(image, text, input_boxes, input_boxes_labels) return list(super().__call__(batch_inputs, **kwargs)) @@ -307,8 +307,7 @@ def preprocess(self, inputs, timeout=None): input_boxes=input_boxes, input_boxes_labels=input_boxes_labels, return_tensors="pt", - ) - model_inputs = model_inputs.to(self.dtype) + ).to(self.dtype) # Store original size for post-processing target_size = torch.tensor([[image.height, image.width]], dtype=torch.int32) @@ -373,18 +372,9 @@ def postprocess(self, model_outputs, threshold=0.3, mask_threshold=0.5, top_k=No box_tensor = results["boxes"][i] mask_tensor = results["masks"][i] - # Convert box to dict format - xmin, ymin, xmax, ymax = box_tensor.int().tolist() - box_dict = { - "xmin": xmin, - "ymin": ymin, - "xmax": xmax, - "ymax": ymax, - } - result = { "score": score, - "box": box_dict, + "box": self._get_bounding_box(box_tensor), "mask": mask_tensor, } @@ -402,3 +392,12 @@ def postprocess(self, model_outputs, threshold=0.3, mask_threshold=0.5, top_k=No final_results = final_results[:top_k] return final_results + + def _get_bounding_box(self, box: "torch.Tensor") -> dict[str, int]: + xmin, ymin, xmax, ymax = box.int().tolist() + return { + "xmin": xmin, + "ymin": ymin, + "xmax": xmax, + "ymax": ymax, + } diff --git a/tests/models/sam3/test_modeling_sam3.py b/tests/models/sam3/test_modeling_sam3.py index ae20e522d260..8ec3ba017470 100644 --- a/tests/models/sam3/test_modeling_sam3.py +++ b/tests/models/sam3/test_modeling_sam3.py @@ -423,7 +423,9 @@ class Sam3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): """ all_model_classes = (Sam3Model,) if is_torch_available() else () - pipeline_model_mapping = {"mask-generation": Sam3Model} if is_torch_available() else {} + pipeline_model_mapping = ( + {"mask-generation": Sam3Model, "promptable-concept-segmentation": Sam3Model} if is_torch_available() else {} + ) test_resize_embeddings = False _is_composite = True diff --git a/utils/check_docstrings.py b/utils/check_docstrings.py index ea03cfe4e48d..b91b7a138ac0 100644 --- a/utils/check_docstrings.py +++ b/utils/check_docstrings.py @@ -109,6 +109,7 @@ class DecoratedItem: "SmolLM3Config", "Gemma3nVisionConfig", "Llama4Processor", + "PromptableConceptSegmentationPipeline", # Deprecated "InputExample", "InputFeatures", From f8d3f2ce86e4b770800c9fb017bfcd3ca1caa058 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Fri, 30 Jan 2026 19:33:57 +0000 Subject: [PATCH 05/10] change back --- .../models/pp_doclayout_v3/modeling_pp_doclayout_v3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py index d717c595ab9e..3891a23dfe89 100644 --- a/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py +++ b/src/transformers/models/pp_doclayout_v3/modeling_pp_doclayout_v3.py @@ -1987,15 +1987,15 @@ class PPDocLayoutV3ForObjectDetectionOutput(ModelOutput): r""" logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): Classification logits (including no-object) for all queries. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits of the final layer of the decoder. + out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): + Masks of the final layer of the decoder. pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). You can use [`~PPDocLayoutV3ImageProcessorFast.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. - order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Order logits of the final layer of the decoder. - out_masks (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, height, width)`): - Masks of the final layer of the decoder. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): From 6e422151913337578ff0042548cd6c578fda4378 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Fri, 30 Jan 2026 23:50:24 +0000 Subject: [PATCH 06/10] fix docs --- docs/source/en/main_classes/pipelines.md | 6 ++++++ docs/source/en/model_doc/auto.md | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/docs/source/en/main_classes/pipelines.md b/docs/source/en/main_classes/pipelines.md index e2a40ac3cc3e..e0a207d24c61 100644 --- a/docs/source/en/main_classes/pipelines.md +++ b/docs/source/en/main_classes/pipelines.md @@ -473,6 +473,12 @@ Pipelines available for multimodal tasks include the following. - __call__ - all +### PromptableConceptSegmentationPipeline + +[[autodoc]] PromptableConceptSegmentationPipeline + - __call__ + - all + ### VisualQuestionAnsweringPipeline [[autodoc]] VisualQuestionAnsweringPipeline diff --git a/docs/source/en/model_doc/auto.md b/docs/source/en/model_doc/auto.md index b45b3bfdb187..9a61e98aeb54 100644 --- a/docs/source/en/model_doc/auto.md +++ b/docs/source/en/model_doc/auto.md @@ -113,6 +113,10 @@ The following auto classes are available for the following natural language proc [[autodoc]] AutoModelForMaskGeneration +### AutoModelForPromptableConceptSegmentation + +[[autodoc]] AutoModelForPromptableConceptSegmentation + ### AutoModelForSeq2SeqLM [[autodoc]] AutoModelForSeq2SeqLM From 817b77d37d563086f553af98d5553829752f635b Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Sat, 31 Jan 2026 00:09:59 +0000 Subject: [PATCH 07/10] fix metadata --- utils/update_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/update_metadata.py b/utils/update_metadata.py index 231380df8f58..69c5856cf8be 100755 --- a/utils/update_metadata.py +++ b/utils/update_metadata.py @@ -74,6 +74,11 @@ "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForZeroShotObjectDetection", ), + ( + "promptable-concept-segmentation", + "MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING_NAMES", + "AutoModelForPromptableConceptSegmentation", + ), ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"), ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"), ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"), From 0ff1134591d55cb9889b5d7e274859d9e3d2deb1 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Sat, 31 Jan 2026 00:16:37 +0000 Subject: [PATCH 08/10] update toctree --- docs/source/en/_toctree.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 22d127edda1a..f5c1da03aff9 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -335,6 +335,8 @@ title: Video-text-to-text - local: tasks/visual_document_retrieval title: Visual Document Retrieval + - local: promptable_concept_segmentation + title: Promptable Concept Segmentation title: Multimodal title: Task recipes - local: run_scripts From d760aaa78a73c5dd8e77ebf1a940f5a1d21f6086 Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Sat, 31 Jan 2026 00:20:42 +0000 Subject: [PATCH 09/10] nit --- docs/source/en/_toctree.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f5c1da03aff9..b6f32afcbc41 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -335,7 +335,7 @@ title: Video-text-to-text - local: tasks/visual_document_retrieval title: Visual Document Retrieval - - local: promptable_concept_segmentation + - local: tasks/promptable_concept_segmentation title: Promptable Concept Segmentation title: Multimodal title: Task recipes From e062277c3a9e65972100417e74c5ee2a6a836c7a Mon Sep 17 00:00:00 2001 From: yonigozlan Date: Sat, 31 Jan 2026 00:30:14 +0000 Subject: [PATCH 10/10] nit test --- ...ipelines_promptable_concept_segmentation.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/pipelines/test_pipelines_promptable_concept_segmentation.py b/tests/pipelines/test_pipelines_promptable_concept_segmentation.py index c6834c38da81..02066c006074 100644 --- a/tests/pipelines/test_pipelines_promptable_concept_segmentation.py +++ b/tests/pipelines/test_pipelines_promptable_concept_segmentation.py @@ -15,9 +15,9 @@ import unittest from transformers import ( + MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING, PromptableConceptSegmentationPipeline, is_torch_available, - is_vision_available, pipeline, ) from transformers.testing_utils import ( @@ -28,16 +28,6 @@ ) -if is_vision_available(): - from PIL import Image -else: - - class Image: - @staticmethod - def open(*args, **kwargs): - pass - - if is_torch_available(): import torch @@ -46,6 +36,12 @@ def open(*args, **kwargs): @require_vision @require_torch class PromptableConceptSegmentationPipelineTests(unittest.TestCase): + model_mapping = ( + dict(list(MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING.items())) + if MODEL_FOR_PROMPTABLE_CONCEPT_SEGMENTATION_MAPPING + else [] + ) + def get_test_pipeline( self, model,