diff --git a/atom/entrypoints/openai_server.py b/atom/entrypoints/openai_server.py index 2aef6b702e..1fe6efa998 100644 --- a/atom/entrypoints/openai_server.py +++ b/atom/entrypoints/openai_server.py @@ -4,6 +4,7 @@ import time import queue from typing import List, Optional, Dict, Any, AsyncGenerator +from contextlib import asynccontextmanager import uuid from collections import defaultdict @@ -197,7 +198,17 @@ async def generate_async( } -app = FastAPI(title="Atom OpenAI API Server") +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan context manager for startup and shutdown events.""" + # Startup + logger.info("Server started successfully and ready to accept requests") + print("Server started successfully and ready to accept requests!") + yield + # Shutdown (if needed in the future) + + +app = FastAPI(title="Atom OpenAI API Server", lifespan=lifespan) @app.post("/v1/chat/completions") diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index a65705a098..4c2a01a273 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -60,10 +60,6 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.stream_output_queue = queue.Queue() # Queue for streaming intermediate outputs self.input_address = input_address self.output_address = output_address - self.input_thread = threading.Thread( - target=self.process_input_sockets, args=(self.input_address,), daemon=True - ) - self.input_thread.start() self.output_thread = threading.Thread( target=self.process_output_sockets, args=(self.output_address,), daemon=True ) @@ -106,6 +102,13 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.scheduler = Scheduler(config) + # Start input thread AFTER model is loaded so the "ready" signal + # is sent only when the engine is truly ready to accept requests + self.input_thread = threading.Thread( + target=self.process_input_sockets, args=(self.input_address,), daemon=True + ) + self.input_thread.start() + def _init_data_parallel(self, config: Config): pass