Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion atom/entrypoints/openai_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import time
import queue
from typing import List, Optional, Dict, Any, AsyncGenerator
from contextlib import asynccontextmanager
import uuid
from collections import defaultdict

Expand Down Expand Up @@ -197,7 +198,17 @@ async def generate_async(
}


app = FastAPI(title="Atom OpenAI API Server")
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan context manager for startup and shutdown events."""
# Startup
logger.info("Server started successfully and ready to accept requests")
print("Server started successfully and ready to accept requests!")
yield
# Shutdown (if needed in the future)


app = FastAPI(title="Atom OpenAI API Server", lifespan=lifespan)


@app.post("/v1/chat/completions")
Expand Down
11 changes: 7 additions & 4 deletions atom/model_engine/engine_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,6 @@ def __init__(self, config: Config, input_address: str, output_address: str):
self.stream_output_queue = queue.Queue() # Queue for streaming intermediate outputs
self.input_address = input_address
self.output_address = output_address
self.input_thread = threading.Thread(
target=self.process_input_sockets, args=(self.input_address,), daemon=True
)
self.input_thread.start()
self.output_thread = threading.Thread(
target=self.process_output_sockets, args=(self.output_address,), daemon=True
)
Expand Down Expand Up @@ -106,6 +102,13 @@ def __init__(self, config: Config, input_address: str, output_address: str):

self.scheduler = Scheduler(config)

# Start input thread AFTER model is loaded so the "ready" signal
# is sent only when the engine is truly ready to accept requests
self.input_thread = threading.Thread(
target=self.process_input_sockets, args=(self.input_address,), daemon=True
)
self.input_thread.start()

def _init_data_parallel(self, config: Config):
pass

Expand Down