Merge branch 'dev' into release

Author: sethjuarez

api/agent/__init__.py

@@ -107,7 +107,7 @@ async def get_agent(id: str):
 def send_agent_status(connection_id: str, name: str, call_id: str) -> AgentUpdateEvent:
     global connections
 
-    async def status_fn(
+    async def send_status(
         id: str,
         status: str,
         information: str | None = None,
@@ -140,7 +140,7 @@ async def status_fn(
             print(f"Agent {name} ({id}) - {status}")
 
     # return status function
-    return status_fn
+    return send_status
 
 
 class FunctionCall(BaseModel):

api/agent/handler.py

@@ -1,6 +1,7 @@
 import inspect
 import json
 from typing import Any, Union
+from prompty.tracer import trace
 
 from azure.ai.projects.models import (
     AsyncAgentEventHandler,
@@ -31,6 +32,7 @@ def __init__(
         self.project_client = project_client
         self.history: list[dict[str, Any]] = []
 
+    @trace(name="send_agent_status")
     async def add_message(
         self,
         message: Union[ThreadMessage, ThreadRun, RunStep],
@@ -161,14 +163,14 @@ async def execute_tool_call(self, tool_call: RequiredFunctionToolCall) -> Any:
             arguments = json.loads(tool_call.function.arguments)
         except json.JSONDecodeError as e:
             raise ValueError(f"Invalid JSON arguments: {e}") from e
-        
+
         if function_name not in self.tools:
             raise ValueError(f"Function {function_name} not found in tools.")
-        
+
         function = self.tools[function_name]
         if not inspect.iscoroutinefunction(function.func):
             raise ValueError(f"Function {function_name} is not a coroutine function (or awaitable).")
-        
+
         arguments["notify"] = self.notify
         # Implement the logic to execute the tool call here
         # This is a placeholder implementation and should be replaced with actual logic

api/voice/session.py

@@ -106,7 +106,7 @@ async def update_realtime_session(
                     create_response=True,
                     interrupt_response=True,
                 )
-            
+
             elif detection_type == "server_vad":
                 vad = SessionTurnDetection(
                     type=detection_type,
@@ -119,7 +119,7 @@ async def update_realtime_session(
                     f"Invalid detection type: {detection_type}. "
                     "Must be 'semantic_vad' or 'server_vad'."
                 )
-            
+
             session: Session = Session(
                 input_audio_format="pcm16",
                 turn_detection=vad,
@@ -142,12 +142,15 @@ async def update_realtime_session(
     @trace
     async def receive_realtime(self):
         # signature = "api.session.RealtimeSession.receive_realtime"
-        #while self.realtime is not None:
+        # while self.realtime is not None:
         async for event in self.realtime:
             if "delta" not in event.type:
                 print(event.type)
             self.active = True
-            if self.realtime is None or self.connection.state != WebSocketState.CONNECTED:
+            if (
+                self.realtime is None
+                or self.connection.state != WebSocketState.CONNECTED
+            ):
                 break
 
             match event.type:
@@ -167,9 +170,7 @@ async def receive_realtime(self):
                         event
                     )
                 case "conversation.item.input_audio_transcription.delta":
-                    await self._conversation_item_input_audio_transcription_delta(
-                        event
-                    )
+                    await self._conversation_item_input_audio_transcription_delta(event)
                 case "conversation.item.input_audio_transcription.failed":
                     await self._conversation_item_input_audio_transcription_failed(
                         event
@@ -221,7 +222,6 @@ async def receive_realtime(self):
                         f"Unhandled event type {event.type}",
                     )
 
-
     @trace(name="error")
     async def _handle_error(self, event: ErrorEvent):
         print("Error event", event.error)
@@ -249,11 +249,13 @@ async def _conversation_item_input_audio_transcription_completed(
         if event.transcript is None or len(event.transcript.strip()) == 0:
             return
 
-        await self.connection.send_update(Update.message(
-            id=event.item_id,
-            role="user",
-            content=event.transcript.strip(),
-        ))
+        await self.connection.send_update(
+            Update.message(
+                id=event.item_id,
+                role="user",
+                content=event.transcript.strip(),
+            )
+        )
 
         if self.thread_id is not None:
             await create_thread_message(

web/app/routes/app.tsx

@@ -181,7 +181,16 @@ export default function Home() {
         break;
       case "agent":
         effort?.addEffort(serverEvent);
-        if (serverEvent.content) {
+        if(serverEvent.status.toLowerCase().includes("failed")) {
+          await sendRealtime({
+            id: serverEvent.id,
+            type: "function_completion",
+            call_id: serverEvent.call_id,
+            output: `The ${serverEvent.name} has failed. Please let the user know there may be issues with this agent in the service and are happy to help in any other way available to you.`,
+          });
+          break;
+        }
+        if (serverEvent.output && serverEvent.content) {
           addOutput(
             serverEvent.name.toLowerCase().replaceAll(" ", "_"),
             serverEvent.name,