Merge pull request #3687 from pipecat-ai/mb/rtvi-mute-events

Emit RTVI events for user mute/unmute

Author: markbackman

changelog/3687.added.md

@@ -0,0 +1 @@
+- Added `UserMuteStartedFrame` and `UserMuteStoppedFrame` system frames, and corresponding `user-mute-started` / `user-mute-stopped` RTVI messages, so clients can observe when mute strategies activate or deactivate.

src/pipecat/frames/frames.py

@@ -1209,6 +1209,28 @@ class UserStoppedSpeakingFrame(SystemFrame):
     emulated: bool = False
 
 
+@dataclass
+class UserMuteStartedFrame(SystemFrame):
+    """Frame indicating that the user has been muted.
+
+    Emitted when a mute strategy activates, suppressing user frames (audio,
+    transcription, interruption) from propagating through the pipeline.
+    """
+
+    pass
+
+
+@dataclass
+class UserMuteStoppedFrame(SystemFrame):
+    """Frame indicating that the user has been unmuted.
+
+    Emitted when a mute strategy deactivates, allowing user frames to
+    propagate through the pipeline again.
+    """
+
+    pass
+
+
 @dataclass
 class UserSpeakingFrame(SystemFrame):
     """Frame indicating the user is speaking.

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -53,6 +53,8 @@
     TextFrame,
     TranscriptionFrame,
     UserImageRawFrame,
+    UserMuteStartedFrame,
+    UserMuteStoppedFrame,
     UserSpeakingFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
@@ -569,8 +571,10 @@ async def _maybe_mute_frame(self, frame: Frame):
             # Emit mute state change events
             if self._user_is_muted:
                 await self._call_event_handler("on_user_mute_started")
+                await self.broadcast_frame(UserMuteStartedFrame)
             else:
                 await self._call_event_handler("on_user_mute_stopped")
+                await self.broadcast_frame(UserMuteStoppedFrame)
 
         return should_mute_frame
 

src/pipecat/processors/frameworks/rtvi.py

@@ -67,6 +67,8 @@
     TTSStartedFrame,
     TTSStoppedFrame,
     TTSTextFrame,
+    UserMuteStartedFrame,
+    UserMuteStoppedFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
@@ -891,6 +893,20 @@ class RTVIUserStoppedSpeakingMessage(BaseModel):
     type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
 
 
+class RTVIUserMuteStartedMessage(BaseModel):
+    """Message indicating user has been muted."""
+
+    label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
+    type: Literal["user-mute-started"] = "user-mute-started"
+
+
+class RTVIUserMuteStoppedMessage(BaseModel):
+    """Message indicating user has been unmuted."""
+
+    label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
+    type: Literal["user-mute-stopped"] = "user-mute-stopped"
+
+
 class RTVIBotStartedSpeakingMessage(BaseModel):
     """Message indicating bot has started speaking."""
 
@@ -1043,6 +1059,7 @@ class RTVIObserverParams:
     bot_audio_level_enabled: bool = False
     user_llm_enabled: bool = True
     user_speaking_enabled: bool = True
+    user_mute_enabled: bool = True
     user_transcription_enabled: bool = True
     user_audio_level_enabled: bool = False
     metrics_enabled: bool = True
@@ -1218,6 +1235,11 @@ async def on_push_frame(self, data: FramePushed):
             and self._params.user_speaking_enabled
         ):
             await self._handle_interruptions(frame)
+        elif (
+            isinstance(frame, (UserMuteStartedFrame, UserMuteStoppedFrame))
+            and self._params.user_mute_enabled
+        ):
+            await self._handle_user_mute(frame)
         elif (
             isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame))
             and self._params.bot_speaking_enabled
@@ -1349,6 +1371,17 @@ async def _handle_interruptions(self, frame: Frame):
         if message:
             await self.send_rtvi_message(message)
 
+    async def _handle_user_mute(self, frame: Frame):
+        """Handle user mute/unmute frames."""
+        message = None
+        if isinstance(frame, UserMuteStartedFrame):
+            message = RTVIUserMuteStartedMessage()
+        elif isinstance(frame, UserMuteStoppedFrame):
+            message = RTVIUserMuteStoppedMessage()
+
+        if message:
+            await self.send_rtvi_message(message)
+
     async def _handle_bot_speaking(self, frame: Frame):
         """Handle bot speaking event frames."""
         message = None