Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions python/sglang/srt/entrypoints/openai/serving_base.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import asyncio
import json
import logging
import uuid
Expand Down Expand Up @@ -88,9 +89,11 @@ async def handle_request(
if request_logger.log_requests and request_logger.log_requests_level >= 2:
request_logger.log_openai_received_request(request, request=raw_request)

# Convert to internal format
adapted_request, processed_request = self._convert_to_internal_request(
request, raw_request
# Convert off-loop: jinja render + full-prompt tokenization blocks
# the loop on long prompts, and the rust tokenizer releases the GIL.
loop = asyncio.get_running_loop()
adapted_request, processed_request = await loop.run_in_executor(
None, self._convert_to_internal_request, request, raw_request
)

if isinstance(adapted_request, (GenerateReqInput, EmbeddingReqInput)):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -480,21 +480,25 @@ async def test_questions_yield_to_other_requests(self):
):
with self.subTest(route):
events = []
encoding_started = asyncio.Event()
encode = handler._encode_question

def recorded(**kwargs):
events.append("encode")
encoding_started.set()
return encode(**kwargs)

async def other_request():
# Conversion may now yield before question encoding starts.
await encoding_started.wait()
for _ in range(3):
events.append("other")
await asyncio.sleep(0)

handler._encode_question = recorded
other = asyncio.create_task(other_request())
response = await handler.handle_request(request, None)
await other
await asyncio.wait_for(other, timeout=5)
self.assertEqual(response.status_code, 200)
self.assertEqual(events, ["encode", "other"] * 3)

Expand Down
Loading