Perception Encoder follow up PR

docs/source/en/model_doc/pe_audio_video.md

@@ -26,7 +26,47 @@ TODO
 ### Basic usage
 
 ```py
-TODO
+
+model = PeAudioVideoModel.from_pretrained("facebook/pe-av-large", device_map="cuda", dtype=torch.bfloat16)
+processor = PeAudioVideoProcessor.from_pretrained("facebook/pe-av-large")
+
+from huggingface_hub import hf_hub_download
+
+video_path = hf_hub_download(
+    repo_id="eustlb/dummy-video-dataset", filename="audiobox.mp4", repo_type="dataset"
+)
+
+video_path2 = hf_hub_download(
+    repo_id="eustlb/dummy-video-dataset", filename="glass_breaking.mp4", repo_type="dataset"
+)
+
+audio_path = hf_hub_download(
+    repo_id="eustlb/dummy-video-dataset", filename="audiobox.mp4", repo_type="dataset"
+)
+
+audio_path2 = hf_hub_download(
+    repo_id="eustlb/dummy-video-dataset", filename="glass_breaking.mp4", repo_type="dataset"
+)
+
+video_files = [video_path, video_path2]
+descriptions = ["A woman and a man speaking", "A glass breaking"]
+audio_files = [audio_path, audio_path2]
+
+inputs = processor(
+    videos=video_files, text=descriptions, audio=audio_files, return_tensors="pt", padding=True
+)
+
+with torch.inference_mode(), torch.autocast(model.device.type, dtype=torch.bfloat16):
+    outputs = model(**inputs.to(model.device, dtype=model.dtype))
+
+audio_embeds = outputs.audio_embeds  # Audio-only embeddings
+video_embeds = outputs.video_embeds  # Video-only embeddings
+audio_video_embeds = outputs.audio_video_embeds  # Joint audio-video embeddings
+text_audio_embeds = outputs.text_audio_embeds  # Text embeddings aligned to audio
+text_video_embeds = outputs.text_video_embeds  # Text embeddings aligned to video
+text_audio_video_embeds = outputs.text_audio_video_embeds  # Text embeddings aligned to audio-video
+audio_plus_text_embeds = outputs.audio_plus_text_embeds  # Joint audio and text embedding
+video_plus_text_embeds = outputs.video_plus_text_embeds  # Joint video and text embedding
 ```
 
 ## PeAudioVideoProcessor

src/transformers/conversion_mapping.py

@@ -237,6 +237,16 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
+        "pe_audio_video_encoder": [
+            WeightRenaming(
+                source_patterns=r"audio_model\.audio_encoder\.(.+)",
+                target_patterns=r"embedder.audio_encoder.\1",
+            ),
+            WeightRenaming(
+                source_patterns=r"video_model\.video_encoder\.(.+)",
+                target_patterns=r"embedder.video_encoder.\1",
+            ),
+        ],
         "timm_wrapper": [
             # Simply add the prefix `timm_model`
             # TODO: Would be probably much cleaner with a `add_prefix` argument in WeightRenaming

src/transformers/models/pe_audio/configuration_pe_audio.py

@@ -197,6 +197,7 @@ def __init__(
 
         self.text_config = text_config
         self.audio_config = audio_config
+        self.tie_word_embeddings = True
 
         super().__init__(**kwargs)
 

src/transformers/models/pe_audio_video/configuration_pe_audio_video.py

@@ -202,6 +202,7 @@ def __init__(
 
         self.text_config = text_config
         self.audio_video_config = audio_video_config
+        self.tie_word_embeddings = True
 
         super().__init__(**kwargs)
 

src/transformers/models/pe_video/configuration_pe_video.py

@@ -202,6 +202,7 @@ def __init__(
 
         self.text_config = text_config
         self.video_config = video_config
+        self.tie_word_embeddings = True
 
         super().__init__(**kwargs)