feat: move ollama to new agent loop (#3615)

Author: sarahwooders

letta/llm_api/llm_client.py

@@ -58,7 +58,7 @@ def create(
                     put_inner_thoughts_first=put_inner_thoughts_first,
                     actor=actor,
                 )
-            case ProviderType.openai | ProviderType.together:
+            case ProviderType.openai | ProviderType.together | ProviderType.ollama:
                 from letta.llm_api.openai_client import OpenAIClient
 
                 return OpenAIClient(

letta/schemas/providers/ollama.py

@@ -13,6 +13,8 @@
 
 logger = get_logger(__name__)
 
+ollama_prefix = "/v1"
+
 
 class OllamaProvider(OpenAIProvider):
     """Ollama provider that uses the native /api/generate endpoint
@@ -43,13 +45,13 @@ async def list_llm_models_async(self) -> list[LLMConfig]:
         for model in response_json["models"]:
             context_window = self.get_model_context_window(model["name"])
             if context_window is None:
-                print(f"Ollama model {model['name']} has no context window")
-                continue
+                print(f"Ollama model {model['name']} has no context window, using default 32000")
+                context_window = 32000
             configs.append(
                 LLMConfig(
                     model=model["name"],
-                    model_endpoint_type="ollama",
-                    model_endpoint=self.base_url,
+                    model_endpoint_type=ProviderType.ollama,
+                    model_endpoint=f"{self.base_url}{ollama_prefix}",
                     model_wrapper=self.default_prompt_formatter,
                     context_window=context_window,
                     handle=self.get_handle(model["name"]),
@@ -75,13 +77,14 @@ async def list_embedding_models_async(self) -> list[EmbeddingConfig]:
         for model in response_json["models"]:
             embedding_dim = await self._get_model_embedding_dim_async(model["name"])
             if not embedding_dim:
-                print(f"Ollama model {model['name']} has no embedding dimension")
-                continue
+                print(f"Ollama model {model['name']} has no embedding dimension, using default 1024")
+                # continue
+                embedding_dim = 1024
             configs.append(
                 EmbeddingConfig(
                     embedding_model=model["name"],
-                    embedding_endpoint_type="ollama",
-                    embedding_endpoint=self.base_url,
+                    embedding_endpoint_type=ProviderType.ollama,
+                    embedding_endpoint=f"{self.base_url}{ollama_prefix}",
                     embedding_dim=embedding_dim,
                     embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,
                     handle=self.get_handle(model["name"], is_embedding=True),

letta/server/rest_api/routers/v1/agents.py

@@ -865,7 +865,15 @@ async def send_message(
     # TODO: This is redundant, remove soon
     agent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])
     agent_eligible = agent.multi_agent_group is None or agent.multi_agent_group.manager_type in ["sleeptime", "voice_sleeptime"]
-    model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "together", "google_ai", "google_vertex", "bedrock"]
+    model_compatible = agent.llm_config.model_endpoint_type in [
+        "anthropic",
+        "openai",
+        "together",
+        "google_ai",
+        "google_vertex",
+        "bedrock",
+        "ollama",
+    ]
 
     # Create a new run for execution tracking
     if settings.track_agent_run:
@@ -999,7 +1007,15 @@ async def send_message_streaming(
     # TODO: This is redundant, remove soon
     agent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])
     agent_eligible = agent.multi_agent_group is None or agent.multi_agent_group.manager_type in ["sleeptime", "voice_sleeptime"]
-    model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "together", "google_ai", "google_vertex", "bedrock"]
+    model_compatible = agent.llm_config.model_endpoint_type in [
+        "anthropic",
+        "openai",
+        "together",
+        "google_ai",
+        "google_vertex",
+        "bedrock",
+        "ollama",
+    ]
     model_compatible_token_streaming = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "bedrock"]
     not_letta_endpoint = agent.llm_config.model_endpoint != LETTA_MODEL_ENDPOINT
 
@@ -1194,6 +1210,7 @@ async def _process_message_background(
             "google_ai",
             "google_vertex",
             "bedrock",
+            "ollama",
         ]
         if agent_eligible and model_compatible:
             if agent.enable_sleeptime and agent.agent_type != AgentType.voice_convo_agent:
@@ -1373,7 +1390,15 @@ async def preview_raw_payload(
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
     agent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])
     agent_eligible = agent.multi_agent_group is None or agent.multi_agent_group.manager_type in ["sleeptime", "voice_sleeptime"]
-    model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "together", "google_ai", "google_vertex", "bedrock"]
+    model_compatible = agent.llm_config.model_endpoint_type in [
+        "anthropic",
+        "openai",
+        "together",
+        "google_ai",
+        "google_vertex",
+        "bedrock",
+        "ollama",
+    ]
 
     if agent_eligible and model_compatible:
         if agent.enable_sleeptime:
@@ -1433,7 +1458,15 @@ async def summarize_agent_conversation(
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
     agent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])
     agent_eligible = agent.multi_agent_group is None or agent.multi_agent_group.manager_type in ["sleeptime", "voice_sleeptime"]
-    model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "together", "google_ai", "google_vertex", "bedrock"]
+    model_compatible = agent.llm_config.model_endpoint_type in [
+        "anthropic",
+        "openai",
+        "together",
+        "google_ai",
+        "google_vertex",
+        "bedrock",
+        "ollama",
+    ]
 
     if agent_eligible and model_compatible:
         agent = LettaAgent(

tests/configs/llm_model_configs/ollama.json

@@ -1,7 +1,7 @@
 {
   "context_window": 8192,
   "model_endpoint_type": "ollama",
-  "model_endpoint": "http://127.0.0.1:11434",
-  "model": "qwen3:32b",
+  "model_endpoint": "http://127.0.0.1:11434/v1",
+  "model": "qwen2.5:7b",
   "put_inner_thoughts_in_kwargs": true
 }