From 7aacecc787dd5377211de97d8713bc2e655fe19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 1 Oct 2025 23:06:49 +0000 Subject: [PATCH 1/3] account for token_type_ids in `DataCollatorForVisionLanguageModeling` --- trl/trainer/sft_trainer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/trl/trainer/sft_trainer.py b/trl/trainer/sft_trainer.py index c0f26bc9373..0362490c00d 100644 --- a/trl/trainer/sft_trainer.py +++ b/trl/trainer/sft_trainer.py @@ -418,15 +418,26 @@ def _collate_prompt_completion(self, examples: list[dict[str, Any]]) -> dict[str input_ids = torch.cat((prompt_ids, completion_ids), dim=1) attention_mask = torch.cat((prompt_mask, completion_mask), dim=1) completion_mask = torch.cat((torch.zeros_like(prompt_mask), completion_mask), dim=1) + if "token_type_ids" in processed_prompts: # special case for Gemma + prompt_token_type_ids = processed_prompts["token_type_ids"] + completion_token_type_ids = processed_completions["token_type_ids"] + token_type_ids = torch.cat((prompt_token_type_ids, completion_token_type_ids), dim=1) # Flush left to reduce padding - attention_mask, input_ids, completion_mask = flush_left(attention_mask, input_ids, completion_mask) + if "token_type_ids" in processed_prompts: + attention_mask, input_ids, completion_mask, token_type_ids = flush_left( + attention_mask, input_ids, completion_mask, token_type_ids + ) + else: + attention_mask, input_ids, completion_mask = flush_left(attention_mask, input_ids, completion_mask) # Truncate if necessary if self.max_length is not None: input_ids = input_ids[:, : self.max_length] attention_mask = attention_mask[:, : self.max_length] completion_mask = completion_mask[:, : self.max_length] + if "token_type_ids" in processed_prompts: + token_type_ids = token_type_ids[:, : self.max_length] # Create labels and mask padding tokens labels = input_ids.clone() @@ -439,6 +450,8 @@ def _collate_prompt_completion(self, examples: list[dict[str, Any]]) -> dict[str output["input_ids"] = input_ids output["attention_mask"] = attention_mask output["labels"] = labels + if "token_type_ids" in processed_prompts: + output["token_type_ids"] = token_type_ids return output From 6c30f9068ebf963cf67b520df27fbe0d643557ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 1 Oct 2025 23:20:31 +0000 Subject: [PATCH 2/3] add test --- tests/test_sft_trainer.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index 19d5a1b5d70..1c6ac9ce99c 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -1337,6 +1337,39 @@ def test_train_vlm_prompt_completion(self): new_param = trainer.model.get_parameter(n) self.assertFalse(torch.allclose(param, new_param, rtol=1e-12, atol=1e-12), f"Param {n} is not updated") + # Special case for Gemma, as it uses token_type_ids, and we need to ensure they are properly in the collator. + @require_vision + def test_train_vlm_prompt_completion_gemma(self): + # Get the dataset + dataset = load_dataset("trl-internal-testing/zen-image", "conversational_prompt_completion", split="train") + + # Initialize the trainer + training_args = SFTConfig( + output_dir=self.tmp_dir, + max_length=None, # For VLMs, truncating can remove image tokens, leading to errors + report_to="none", + ) + trainer = SFTTrainer( + model="trl-internal-testing/tiny-Gemma3ForConditionalGeneration", + args=training_args, + train_dataset=dataset, + ) + + # Save the initial parameters to compare them later + previous_trainable_params = {n: param.clone() for n, param in trainer.model.named_parameters()} + + # Train the model + trainer.train() + + # Check that the training loss is not None + self.assertIsNotNone(trainer.state.log_history[-1]["train_loss"]) + + # Check the params have changed + for n, param in previous_trainable_params.items(): + new_param = trainer.model.get_parameter(n) + self.assertFalse(torch.allclose(param, new_param, rtol=1e-12, atol=1e-12), f"Param {n} is not updated") + + # Gemma 3n uses a timm encoder, making it difficult to create a smaller variant for testing. # To ensure coverage, we run tests on the full model but mark them as slow to exclude from default runs. @pytest.mark.slow From d563c293336703b79bdceabc2c7c3fcbe2ffb9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Wed, 1 Oct 2025 23:21:37 +0000 Subject: [PATCH 3/3] style --- tests/test_sft_trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_sft_trainer.py b/tests/test_sft_trainer.py index 1c6ac9ce99c..6b17ab8fabf 100644 --- a/tests/test_sft_trainer.py +++ b/tests/test_sft_trainer.py @@ -1369,7 +1369,6 @@ def test_train_vlm_prompt_completion_gemma(self): new_param = trainer.model.get_parameter(n) self.assertFalse(torch.allclose(param, new_param, rtol=1e-12, atol=1e-12), f"Param {n} is not updated") - # Gemma 3n uses a timm encoder, making it difficult to create a smaller variant for testing. # To ensure coverage, we run tests on the full model but mark them as slow to exclude from default runs. @pytest.mark.slow