From 429a1e24b2d91bf58c3920a8c177746866fbccc4 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Mon, 18 Dec 2023 22:57:05 +0100 Subject: [PATCH 01/16] refactor to separate Uvicorn and llm generation --- vllm/entrypoints/openai/serving.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 vllm/entrypoints/openai/serving.py diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py new file mode 100644 index 000000000000..e69de29bb2d1 From fad02148266c595d6cf13b10b8531effe0cd375a Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Mon, 18 Dec 2023 23:01:10 +0100 Subject: [PATCH 02/16] refactor to separate Uvicorn and llm generation --- vllm/entrypoints/openai/api_server.py | 648 +------------------------- vllm/entrypoints/openai/serving.py | 626 +++++++++++++++++++++++++ vllm/sampling_params.py | 2 +- 3 files changed, 645 insertions(+), 631 deletions(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index be5f4190e633..3252005f491c 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -2,46 +2,27 @@ # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py import argparse -import asyncio -import codecs import json -import time -from http import HTTPStatus -from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union - from aioprometheus import MetricsMiddleware from aioprometheus.asgi.starlette import metrics import fastapi import uvicorn from fastapi import Request -from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, StreamingResponse, Response - +from fastapi.exceptions import RequestValidationError from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.async_llm_engine import AsyncLLMEngine from vllm.engine.metrics import add_global_metrics_labels -from vllm.entrypoints.openai.protocol import ( - CompletionRequest, CompletionResponse, CompletionResponseChoice, - CompletionResponseStreamChoice, CompletionStreamResponse, - ChatCompletionRequest, ChatCompletionResponse, - ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse, - LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo) +from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse +from vllm.entrypoints.openai.serving import OpenAIServing from vllm.logger import init_logger -from vllm.outputs import RequestOutput -from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.utils import random_uuid TIMEOUT_KEEP_ALIVE = 5 # seconds +openai_serving: OpenAIServing = None logger = init_logger(__name__) -served_model = None app = fastapi.FastAPI() -engine = None -response_role = None - def parse_args(): parser = argparse.ArgumentParser( @@ -97,620 +78,36 @@ def parse_args(): app.add_route("/metrics", metrics) # Exposes HTTP metrics -def create_error_response(status_code: HTTPStatus, - message: str) -> JSONResponse: - return JSONResponse(ErrorResponse(message=message, - type="invalid_request_error").dict(), - status_code=status_code.value) - - -def load_chat_template(args, tokenizer): - if args.chat_template is not None: - try: - with open(args.chat_template, "r") as f: - chat_template = f.read() - except OSError: - # If opening a file fails, set chat template to be args to - # ensure we decode so our escape are interpreted correctly - chat_template = codecs.decode(args.chat_template, "unicode_escape") - - tokenizer.chat_template = chat_template - logger.info( - f"Using supplied chat template:\n{tokenizer.chat_template}") - elif tokenizer.chat_template is not None: - logger.info(f"Using default chat template:\n{tokenizer.chat_template}") - else: - logger.warning("No chat template provided. Chat API will not work.") - - @app.exception_handler(RequestValidationError) async def validation_exception_handler(_, exc): - return create_error_response(HTTPStatus.BAD_REQUEST, str(exc)) - - -async def check_model(request) -> Optional[JSONResponse]: - if request.model == served_model: - return - ret = create_error_response( - HTTPStatus.NOT_FOUND, - f"The model `{request.model}` does not exist.", - ) - return ret - - -async def check_length( - request: Union[ChatCompletionRequest, CompletionRequest], - prompt: Optional[str] = None, - prompt_ids: Optional[List[int]] = None -) -> Tuple[List[int], Optional[JSONResponse]]: - assert (not (prompt is None and prompt_ids is None) - and not (prompt is not None and prompt_ids is not None) - ), "Either prompt or prompt_ids should be provided." - input_ids = prompt_ids if prompt_ids is not None else tokenizer( - prompt).input_ids - token_num = len(input_ids) - - if request.max_tokens is None: - request.max_tokens = max_model_len - token_num - if token_num + request.max_tokens > max_model_len: - return input_ids, create_error_response( - HTTPStatus.BAD_REQUEST, - f"This model's maximum context length is {max_model_len} tokens. " - f"However, you requested {request.max_tokens + token_num} tokens " - f"({token_num} in the messages, " - f"{request.max_tokens} in the completion). " - f"Please reduce the length of the messages or completion.", - ) - else: - return input_ids, None - + return openai_serving.create_error_response(message=str(exc)) @app.get("/health") async def health() -> Response: """Health check.""" return Response(status_code=200) - @app.get("/v1/models") async def show_available_models(): - """Show available models. Right now we only have one model.""" - model_cards = [ - ModelCard(id=served_model, - root=served_model, - permission=[ModelPermission()]) - ] - return ModelList(data=model_cards) - - -def create_logprobs( - token_ids: List[int], - top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None, - num_output_top_logprobs: Optional[int] = None, - initial_text_offset: int = 0, -) -> LogProbs: - """Create OpenAI-style logprobs.""" - logprobs = LogProbs() - last_token_len = 0 - if num_output_top_logprobs: - logprobs.top_logprobs = [] - for i, token_id in enumerate(token_ids): - step_top_logprobs = top_logprobs[i] - if step_top_logprobs is not None: - token_logprob = step_top_logprobs[token_id] - else: - token_logprob = None - token = tokenizer.convert_ids_to_tokens(token_id) - logprobs.tokens.append(token) - logprobs.token_logprobs.append(token_logprob) - if len(logprobs.text_offset) == 0: - logprobs.text_offset.append(initial_text_offset) - else: - logprobs.text_offset.append(logprobs.text_offset[-1] + - last_token_len) - last_token_len = len(token) - - if num_output_top_logprobs: - logprobs.top_logprobs.append({ - tokenizer.convert_ids_to_tokens(i): p - for i, p in step_top_logprobs.items() - } if step_top_logprobs else None) - return logprobs - + models = await openai_serving.show_available_models() + return JSONResponse(content=models.dict()) + pass @app.post("/v1/chat/completions") -async def create_chat_completion(request: ChatCompletionRequest, - raw_request: Request): - """Completion API similar to OpenAI's API. - - See https://platform.openai.com/docs/api-reference/chat/create - for the API specification. This API mimics the OpenAI ChatCompletion API. - - NOTE: Currently we do not support the following features: - - function_call (Users should implement this by themselves) - - logit_bias (to be supported by vLLM engine) - """ - error_check_ret = await check_model(request) - if error_check_ret is not None: - return error_check_ret - - if request.logit_bias is not None and len(request.logit_bias) > 0: - # TODO: support logit_bias in vLLM engine. - return create_error_response(HTTPStatus.BAD_REQUEST, - "logit_bias is not currently supported") - - try: - prompt = tokenizer.apply_chat_template( - conversation=request.messages, - tokenize=False, - add_generation_prompt=request.add_generation_prompt) - except Exception as e: - logger.error(f"Error in applying chat template from request: {str(e)}") - return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) - - token_ids, error_check_ret = await check_length(request, prompt=prompt) - if error_check_ret is not None: - return error_check_ret - - model_name = request.model - request_id = f"cmpl-{random_uuid()}" - created_time = int(time.monotonic()) - chunk_object_type = "chat.completion.chunk" - try: - spaces_between_special_tokens = request.spaces_between_special_tokens - sampling_params = SamplingParams( - n=request.n, - presence_penalty=request.presence_penalty, - frequency_penalty=request.frequency_penalty, - repetition_penalty=request.repetition_penalty, - temperature=request.temperature, - top_p=request.top_p, - min_p=request.min_p, - stop=request.stop, - stop_token_ids=request.stop_token_ids, - max_tokens=request.max_tokens, - best_of=request.best_of, - top_k=request.top_k, - ignore_eos=request.ignore_eos, - use_beam_search=request.use_beam_search, - skip_special_tokens=request.skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - except ValueError as e: - return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) - - result_generator = engine.generate(prompt, sampling_params, request_id, - token_ids) - - def get_role() -> str: - if request.add_generation_prompt: - return response_role - else: - return request.messages[-1]["role"] - - async def completion_stream_generator() -> AsyncGenerator[str, None]: - # Send first response for each request.n (index) with the role - role = get_role() - for i in range(request.n): - choice_data = ChatCompletionResponseStreamChoice( - index=i, delta=DeltaMessage(role=role), finish_reason=None) - chunk = ChatCompletionStreamResponse(id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - data = chunk.json(exclude_unset=True, ensure_ascii=False) - yield f"data: {data}\n\n" - - # Send response to echo the input portion of the last message - if request.echo: - last_msg_content = "" - if request.messages and isinstance( - request.messages, list) and request.messages[-1].get( - "content") and request.messages[-1].get( - "role") == role: - last_msg_content = request.messages[-1]["content"] - if last_msg_content: - for i in range(request.n): - choice_data = ChatCompletionResponseStreamChoice( - index=i, - delta=DeltaMessage(content=last_msg_content), - finish_reason=None) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - data = chunk.json(exclude_unset=True, ensure_ascii=False) - yield f"data: {data}\n\n" - - # Send response for each token for each request.n (index) - previous_texts = [""] * request.n - previous_num_tokens = [0] * request.n - finish_reason_sent = [False] * request.n - async for res in result_generator: - res: RequestOutput - for output in res.outputs: - i = output.index - - if finish_reason_sent[i]: - continue - - if output.finish_reason is None: - # Send token-by-token response for each request.n - delta_text = output.text[len(previous_texts[i]):] - previous_texts[i] = output.text - previous_num_tokens[i] = len(output.token_ids) - choice_data = ChatCompletionResponseStreamChoice( - index=i, - delta=DeltaMessage(content=delta_text), - finish_reason=None) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - data = chunk.json(exclude_unset=True, ensure_ascii=False) - yield f"data: {data}\n\n" - else: - # Send the finish response for each request.n only once - prompt_tokens = len(res.prompt_token_ids) - final_usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=previous_num_tokens[i], - total_tokens=prompt_tokens + previous_num_tokens[i], - ) - choice_data = ChatCompletionResponseStreamChoice( - index=i, delta=[], finish_reason=output.finish_reason) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - if final_usage is not None: - chunk.usage = final_usage - data = chunk.json(exclude_unset=True, - exclude_none=True, - ensure_ascii=False) - yield f"data: {data}\n\n" - finish_reason_sent[i] = True - # Send the final done message after all response.n are finished - yield "data: [DONE]\n\n" - - async def completion_full_generator(): - final_res: RequestOutput = None - async for res in result_generator: - if await raw_request.is_disconnected(): - # Abort the request if the client disconnects. - await engine.abort(request_id) - return create_error_response(HTTPStatus.BAD_REQUEST, - "Client disconnected") - final_res = res - assert final_res is not None - - choices = [] - role = get_role() - for output in final_res.outputs: - choice_data = ChatCompletionResponseChoice( - index=output.index, - message=ChatMessage(role=role, content=output.text), - finish_reason=output.finish_reason, - ) - choices.append(choice_data) - - if request.echo: - last_msg_content = "" - if request.messages and isinstance( - request.messages, list) and request.messages[-1].get( - "content") and request.messages[-1].get( - "role") == role: - last_msg_content = request.messages[-1]["content"] - - for choice in choices: - full_message = last_msg_content + choice.message.content - choice.message.content = full_message - - num_prompt_tokens = len(final_res.prompt_token_ids) - num_generated_tokens = sum( - len(output.token_ids) for output in final_res.outputs) - usage = UsageInfo( - prompt_tokens=num_prompt_tokens, - completion_tokens=num_generated_tokens, - total_tokens=num_prompt_tokens + num_generated_tokens, - ) - response = ChatCompletionResponse( - id=request_id, - created=created_time, - model=model_name, - choices=choices, - usage=usage, - ) - - return response - - # Streaming response - if request.stream: - return StreamingResponse(completion_stream_generator(), - media_type="text/event-stream") +async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request): + generator = await openai_serving.create_chat_completion(request, raw_request) + if request.stream and not isinstance(generator, ErrorResponse): + return StreamingResponse(content=generator, media_type="text/event-stream") else: - return await completion_full_generator() - + return JSONResponse(content=generator.dict()) @app.post("/v1/completions") async def create_completion(request: CompletionRequest, raw_request: Request): - """Completion API similar to OpenAI's API. - - See https://platform.openai.com/docs/api-reference/completions/create - for the API specification. This API mimics the OpenAI Completion API. - - NOTE: Currently we do not support the following features: - - suffix (the language models we currently support do not support - suffix) - - logit_bias (to be supported by vLLM engine) - """ - - error_check_ret = await check_model(request) - if error_check_ret is not None: - return error_check_ret - - # OpenAI API supports echoing the prompt when max_tokens is 0. - echo_without_generation = request.echo and request.max_tokens == 0 - - if request.suffix is not None: - # The language models we currently support do not support suffix. - return create_error_response(HTTPStatus.BAD_REQUEST, - "suffix is not currently supported") - - if request.logit_bias is not None and len(request.logit_bias) > 0: - # TODO: support logit_bias in vLLM engine. - return create_error_response(HTTPStatus.BAD_REQUEST, - "logit_bias is not currently supported") - - model_name = request.model - request_id = f"cmpl-{random_uuid()}" - - use_token_ids = False - if isinstance(request.prompt, list): - if len(request.prompt) == 0: - return create_error_response(HTTPStatus.BAD_REQUEST, - "please provide at least one prompt") - first_element = request.prompt[0] - if isinstance(first_element, int): - use_token_ids = True - prompt = request.prompt - elif isinstance(first_element, (str, list)): - # TODO: handles multiple prompt case in list[list[int]] - if len(request.prompt) > 1: - return create_error_response( - HTTPStatus.BAD_REQUEST, - "multiple prompts in a batch is not currently supported") - use_token_ids = not isinstance(first_element, str) - prompt = request.prompt[0] - else: - prompt = request.prompt - - if use_token_ids: - _, error_check_ret = await check_length(request, prompt_ids=prompt) + generator = await openai_serving.create_completion(request, raw_request) + if request.stream and not isinstance(generator, ErrorResponse): + return StreamingResponse(content=generator, media_type="text/event-stream") else: - token_ids, error_check_ret = await check_length(request, prompt=prompt) - if error_check_ret is not None: - return error_check_ret - - created_time = int(time.monotonic()) - try: - spaces_between_special_tokens = request.spaces_between_special_tokens - sampling_params = SamplingParams( - n=request.n, - best_of=request.best_of, - presence_penalty=request.presence_penalty, - frequency_penalty=request.frequency_penalty, - repetition_penalty=request.repetition_penalty, - temperature=request.temperature, - top_p=request.top_p, - top_k=request.top_k, - min_p=request.min_p, - stop=request.stop, - stop_token_ids=request.stop_token_ids, - ignore_eos=request.ignore_eos, - max_tokens=request.max_tokens - if not echo_without_generation else 1, - logprobs=request.logprobs, - use_beam_search=request.use_beam_search, - prompt_logprobs=request.logprobs if request.echo else None, - skip_special_tokens=request.skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - except ValueError as e: - return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) - - if use_token_ids: - result_generator = engine.generate(None, - sampling_params, - request_id, - prompt_token_ids=prompt) - else: - result_generator = engine.generate(prompt, sampling_params, request_id, - token_ids) - - # Similar to the OpenAI API, when n != best_of, we do not stream the - # results. In addition, we do not stream the results when use beam search. - stream = (request.stream - and (request.best_of is None or request.n == request.best_of) - and not request.use_beam_search) - - def create_stream_response_json( - index: int, - text: str, - logprobs: Optional[LogProbs] = None, - finish_reason: Optional[str] = None, - usage: Optional[UsageInfo] = None, - ) -> str: - choice_data = CompletionResponseStreamChoice( - index=index, - text=text, - logprobs=logprobs, - finish_reason=finish_reason, - ) - response = CompletionStreamResponse( - id=request_id, - created=created_time, - model=model_name, - choices=[choice_data], - ) - if usage is not None: - response.usage = usage - response_json = response.json(exclude_unset=True, ensure_ascii=False) - - return response_json - - async def completion_stream_generator() -> AsyncGenerator[str, None]: - previous_texts = [""] * request.n - previous_num_tokens = [0] * request.n - has_echoed = [False] * request.n - async for res in result_generator: - res: RequestOutput - for output in res.outputs: - i = output.index - delta_text = output.text[len(previous_texts[i]):] - token_ids = output.token_ids[previous_num_tokens[i]:] - if request.logprobs is not None: - top_logprobs = output.logprobs[previous_num_tokens[i]:] - else: - top_logprobs = None - offsets = len(previous_texts[i]) - if request.echo and not has_echoed[i]: - if not echo_without_generation: - delta_text = res.prompt + delta_text - token_ids = res.prompt_token_ids + token_ids - if top_logprobs: - top_logprobs = res.prompt_logprobs + top_logprobs - else: # only just return the prompt - delta_text = res.prompt - token_ids = res.prompt_token_ids - if top_logprobs: - top_logprobs = res.prompt_logprobs - has_echoed[i] = True - if request.logprobs is not None: - logprobs = create_logprobs( - token_ids=token_ids, - top_logprobs=top_logprobs, - num_output_top_logprobs=request.logprobs, - initial_text_offset=offsets, - ) - else: - logprobs = None - previous_texts[i] = output.text - previous_num_tokens[i] = len(output.token_ids) - finish_reason = output.finish_reason - response_json = create_stream_response_json( - index=i, - text=delta_text, - logprobs=logprobs, - finish_reason=finish_reason, - ) - yield f"data: {response_json}\n\n" - if output.finish_reason is not None: - logprobs = (LogProbs() - if request.logprobs is not None else None) - prompt_tokens = len(res.prompt_token_ids) - completion_tokens = len(output.token_ids) - final_usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - response_json = create_stream_response_json( - index=i, - text="", - logprobs=logprobs, - finish_reason=output.finish_reason, - usage=final_usage, - ) - yield f"data: {response_json}\n\n" - yield "data: [DONE]\n\n" - - # Streaming response - if stream: - return StreamingResponse(completion_stream_generator(), - media_type="text/event-stream") - - # Non-streaming response - final_res: RequestOutput = None - async for res in result_generator: - if await raw_request.is_disconnected(): - # Abort the request if the client disconnects. - await engine.abort(request_id) - return create_error_response(HTTPStatus.BAD_REQUEST, - "Client disconnected") - final_res = res - assert final_res is not None - choices = [] - prompt_token_ids = final_res.prompt_token_ids - prompt_logprobs = final_res.prompt_logprobs - prompt_text = final_res.prompt - for output in final_res.outputs: - if request.logprobs is not None: - if not echo_without_generation: - token_ids = output.token_ids - top_logprobs = output.logprobs - if request.echo: - token_ids = prompt_token_ids + token_ids - top_logprobs = prompt_logprobs + top_logprobs - else: - token_ids = prompt_token_ids - top_logprobs = prompt_logprobs - logprobs = create_logprobs( - token_ids=token_ids, - top_logprobs=top_logprobs, - num_output_top_logprobs=request.logprobs, - ) - else: - logprobs = None - if not echo_without_generation: - output_text = output.text - if request.echo: - output_text = prompt_text + output_text - else: - output_text = prompt_text - choice_data = CompletionResponseChoice( - index=output.index, - text=output_text, - logprobs=logprobs, - finish_reason=output.finish_reason, - ) - choices.append(choice_data) - - num_prompt_tokens = len(final_res.prompt_token_ids) - num_generated_tokens = sum( - len(output.token_ids) for output in final_res.outputs) - usage = UsageInfo( - prompt_tokens=num_prompt_tokens, - completion_tokens=num_generated_tokens, - total_tokens=num_prompt_tokens + num_generated_tokens, - ) - response = CompletionResponse( - id=request_id, - created=created_time, - model=model_name, - choices=choices, - usage=usage, - ) - - if request.stream: - # When user requests streaming but we don't stream, we still need to - # return a streaming response with a single event. - response_json = response.json(ensure_ascii=False) - - async def fake_stream_generator() -> AsyncGenerator[str, None]: - yield f"data: {response_json}\n\n" - yield "data: [DONE]\n\n" - - return StreamingResponse(fake_stream_generator(), - media_type="text/event-stream") - - return response + return JSONResponse(content=generator.dict()) if __name__ == "__main__": @@ -731,19 +128,10 @@ async def fake_stream_generator() -> AsyncGenerator[str, None]: else: served_model = args.model - response_role = args.response_role - engine_args = AsyncEngineArgs.from_cli_args(args) engine = AsyncLLMEngine.from_engine_args(engine_args) - engine_model_config = asyncio.run(engine.get_model_config()) - max_model_len = engine_model_config.max_model_len - # A separate tokenizer to map token IDs to strings. - tokenizer = get_tokenizer( - engine_model_config.tokenizer, - tokenizer_mode=engine_model_config.tokenizer_mode, - trust_remote_code=engine_model_config.trust_remote_code) - load_chat_template(args, tokenizer) + openai_serving = OpenAIServing(engine, served_model, args.response_role, args.chat_template) # Register labels for metrics add_global_metrics_labels(model_name=engine_args.model) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index e69de29bb2d1..5104ffe04979 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -0,0 +1,626 @@ +import time +import asyncio +import codecs +from http import HTTPStatus +from fastapi import Request +from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional, Tuple, Union +from vllm.logger import init_logger +from vllm.engine.async_llm_engine import AsyncLLMEngine +from vllm.entrypoints.openai.protocol import ( + CompletionRequest, CompletionResponse, CompletionResponseChoice, + CompletionResponseStreamChoice, CompletionStreamResponse, + ChatCompletionRequest, ChatCompletionResponse, + ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse, + LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo) +from vllm.outputs import RequestOutput +from vllm.sampling_params import SamplingParams +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +class OpenAIServing: + def __init__(self, engine: AsyncLLMEngine, served_mode: str, response_role: str, chat_template=None): + self.engine = engine + self.served_model = served_mode + self.response_role = response_role + + engine_model_config = asyncio.run(engine.get_model_config()) + self.max_model_len = engine_model_config.max_model_len + + # A separate tokenizer to map token IDs to strings. + self.tokenizer = get_tokenizer( + engine_model_config.tokenizer, + tokenizer_mode=engine_model_config.tokenizer_mode, + trust_remote_code=engine_model_config.trust_remote_code) + self._load_chat_template(chat_template) + + async def show_available_models(self) -> ModelList: + """Show available models. Right now we only have one model.""" + return ModelList(data=[ + ModelCard(id=self.served_model, + root=self.served_model, + permission=[ModelPermission()]) + ]) + + async def create_chat_completion(self, request: ChatCompletionRequest, raw_request: Request) -> Union[ErrorResponse, AsyncGenerator[str, None], ChatCompletionResponse]: + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/chat/create + for the API specification. This API mimics the OpenAI ChatCompletion API. + + NOTE: Currently we do not support the following features: + - function_call (Users should implement this by themselves) + - logit_bias (to be supported by vLLM engine) + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + if request.logit_bias is not None and len(request.logit_bias) > 0: + # TODO: support logit_bias in vLLM engine. + return self.create_error_response("logit_bias is not currently supported") + + try: + prompt = self.tokenizer.apply_chat_template( + conversation=request.messages, + tokenize=False, + add_generation_prompt=request.add_generation_prompt) + except Exception as e: + logger.error(f"Error in applying chat template from request: {str(e)}") + return self.create_error_response(str(e)) + + token_ids, error_check_ret = await self._check_length(request, prompt=prompt) + if error_check_ret is not None: + return error_check_ret + + try: + spaces_between_special_tokens = request.spaces_between_special_tokens + sampling_params = SamplingParams( + n=request.n, + presence_penalty=request.presence_penalty, + frequency_penalty=request.frequency_penalty, + repetition_penalty=request.repetition_penalty, + temperature=request.temperature, + top_p=request.top_p, + min_p=request.min_p, + stop=request.stop, + stop_token_ids=request.stop_token_ids, + max_tokens=request.max_tokens, + best_of=request.best_of, + top_k=request.top_k, + ignore_eos=request.ignore_eos, + use_beam_search=request.use_beam_search, + skip_special_tokens=request.skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + except ValueError as e: + return self.create_error_response(str(e)) + + request_id = f"cmpl-{random_uuid()}" + result_generator = self.engine.generate(prompt, sampling_params, request_id, token_ids) + + # Streaming response + if request.stream: + return self.chat_completion_stream_generator(request, result_generator, request_id) + else: + return await self.chat_completion_full_generator(request, raw_request, result_generator, request_id) + + def get_chat_request_role(self, request: ChatCompletionRequest) -> str: + if request.add_generation_prompt: + return self.response_role + else: + return request.messages[-1]["role"] + + async def chat_completion_stream_generator(self, + request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], + request_id: str + ) -> Union[ErrorResponse, AsyncGenerator[str, None]]: + + model_name = request.model + created_time = int(time.monotonic()) + chunk_object_type = "chat.completion.chunk" + + # Send first response for each request.n (index) with the role + role = self.get_chat_request_role(request) + for i in range(request.n): + choice_data = ChatCompletionResponseStreamChoice( + index=i, delta=DeltaMessage(role=role), finish_reason=None) + chunk = ChatCompletionStreamResponse(id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + data = chunk.json(exclude_unset=True, ensure_ascii=False) + yield f"data: {data}\n\n" + + # Send response to echo the input portion of the last message + if request.echo: + last_msg_content = "" + if request.messages and isinstance( + request.messages, list) and request.messages[-1].get( + "content") and request.messages[-1].get( + "role") == role: + last_msg_content = request.messages[-1]["content"] + if last_msg_content: + for i in range(request.n): + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage(content=last_msg_content), + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + data = chunk.json(exclude_unset=True, ensure_ascii=False) + yield f"data: {data}\n\n" + + # Send response for each token for each request.n (index) + previous_texts = [""] * request.n + previous_num_tokens = [0] * request.n + finish_reason_sent = [False] * request.n + async for res in result_generator: + res: RequestOutput + for output in res.outputs: + i = output.index + + if finish_reason_sent[i]: + continue + + if output.finish_reason is None: + # Send token-by-token response for each request.n + delta_text = output.text[len(previous_texts[i]):] + previous_texts[i] = output.text + previous_num_tokens[i] = len(output.token_ids) + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage(content=delta_text), + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + data = chunk.json(exclude_unset=True, ensure_ascii=False) + yield f"data: {data}\n\n" + else: + # Send the finish response for each request.n only once + prompt_tokens = len(res.prompt_token_ids) + final_usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=previous_num_tokens[i], + total_tokens=prompt_tokens + previous_num_tokens[i], + ) + choice_data = ChatCompletionResponseStreamChoice( + index=i, delta=[], finish_reason=output.finish_reason) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if final_usage is not None: + chunk.usage = final_usage + data = chunk.json(exclude_unset=True, + exclude_none=True, + ensure_ascii=False) + yield f"data: {data}\n\n" + finish_reason_sent[i] = True + # Send the final done message after all response.n are finished + yield "data: [DONE]\n\n" + + async def chat_completion_full_generator(self, + request: ChatCompletionRequest, + raw_request: Request, + result_generator: AsyncIterator[RequestOutput], + request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]: + model_name = request.model + created_time = int(time.monotonic()) + + final_res: RequestOutput = None + async for res in result_generator: + if await raw_request.is_disconnected(): + # Abort the request if the client disconnects. + await self.engine.abort(request_id) + return self.create_error_response("Client disconnected") + final_res = res + assert final_res is not None + + choices = [] + role = self.get_chat_request_role(request) + for output in final_res.outputs: + choice_data = ChatCompletionResponseChoice( + index=output.index, + message=ChatMessage(role=role, content=output.text), + finish_reason=output.finish_reason, + ) + choices.append(choice_data) + + if request.echo: + last_msg_content = "" + if request.messages and isinstance( + request.messages, list) and request.messages[-1].get( + "content") and request.messages[-1].get( + "role") == role: + last_msg_content = request.messages[-1]["content"] + + for choice in choices: + full_message = last_msg_content + choice.message.content + choice.message.content = full_message + + num_prompt_tokens = len(final_res.prompt_token_ids) + num_generated_tokens = sum( + len(output.token_ids) for output in final_res.outputs) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + ) + response = ChatCompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=usage, + ) + return response + + async def create_completion(self, request: CompletionRequest, raw_request: Request): + """Completion API similar to OpenAI's API. + + See https://platform.openai.com/docs/api-reference/completions/create + for the API specification. This API mimics the OpenAI Completion API. + + NOTE: Currently we do not support the following features: + - suffix (the language models we currently support do not support + suffix) + - logit_bias (to be supported by vLLM engine) + """ + + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + # OpenAI API supports echoing the prompt when max_tokens is 0. + echo_without_generation = request.echo and request.max_tokens == 0 + + if request.suffix is not None: + # The language models we currently support do not support suffix. + return self.create_error_response("suffix is not currently supported") + + if request.logit_bias is not None and len(request.logit_bias) > 0: + # TODO: support logit_bias in vLLM engine. + return self.create_error_response("logit_bias is not currently supported") + + model_name = request.model + request_id = f"cmpl-{random_uuid()}" + + use_token_ids = False + if isinstance(request.prompt, list): + if len(request.prompt) == 0: + return self.create_error_response("please provide at least one prompt") + first_element = request.prompt[0] + if isinstance(first_element, int): + use_token_ids = True + prompt = request.prompt + elif isinstance(first_element, (str, list)): + # TODO: handles multiple prompt case in list[list[int]] + if len(request.prompt) > 1: + return self.create_error_response("multiple prompts in a batch is not currently supported") + use_token_ids = not isinstance(first_element, str) + prompt = request.prompt[0] + else: + prompt = request.prompt + + if use_token_ids: + _, error_check_ret = await self._check_length(request, prompt_ids=prompt) + else: + token_ids, error_check_ret = await self._check_length(request, prompt=prompt) + if error_check_ret is not None: + return error_check_ret + + created_time = int(time.monotonic()) + try: + spaces_between_special_tokens = request.spaces_between_special_tokens + sampling_params = SamplingParams( + n=request.n, + best_of=request.best_of, + presence_penalty=request.presence_penalty, + frequency_penalty=request.frequency_penalty, + repetition_penalty=request.repetition_penalty, + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + min_p=request.min_p, + stop=request.stop, + stop_token_ids=request.stop_token_ids, + ignore_eos=request.ignore_eos, + max_tokens=request.max_tokens if not echo_without_generation else 1, + logprobs=request.logprobs, + use_beam_search=request.use_beam_search, + prompt_logprobs=request.logprobs if request.echo else None, + skip_special_tokens=request.skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + except ValueError as e: + return self.create_error_response(str(e)) + + if use_token_ids: + result_generator = self.engine.generate(None, + sampling_params, + request_id, + prompt_token_ids=prompt) + else: + result_generator = self.engine.generate(prompt, sampling_params, request_id, + token_ids) + + # Similar to the OpenAI API, when n != best_of, we do not stream the + # results. In addition, we do not stream the results when use beam search. + stream = (request.stream + and (request.best_of is None or request.n == request.best_of) + and not request.use_beam_search) + + def create_stream_response_json( + index: int, + text: str, + logprobs: Optional[LogProbs] = None, + finish_reason: Optional[str] = None, + usage: Optional[UsageInfo] = None, + ) -> str: + choice_data = CompletionResponseStreamChoice( + index=index, + text=text, + logprobs=logprobs, + finish_reason=finish_reason, + ) + response = CompletionStreamResponse( + id=request_id, + created=created_time, + model=model_name, + choices=[choice_data], + ) + if usage is not None: + response.usage = usage + response_json = response.json(exclude_unset=True, ensure_ascii=False) + + return response_json + + async def completion_stream_generator() -> AsyncGenerator[str, None]: + previous_texts = [""] * request.n + previous_num_tokens = [0] * request.n + has_echoed = [False] * request.n + async for res in result_generator: + res: RequestOutput + for output in res.outputs: + i = output.index + delta_text = output.text[len(previous_texts[i]):] + token_ids = output.token_ids[previous_num_tokens[i]:] + if request.logprobs is not None: + top_logprobs = output.logprobs[previous_num_tokens[i]:] + else: + top_logprobs = None + offsets = len(previous_texts[i]) + if request.echo and not has_echoed[i]: + if not echo_without_generation: + delta_text = res.prompt + delta_text + token_ids = res.prompt_token_ids + token_ids + if top_logprobs: + top_logprobs = res.prompt_logprobs + top_logprobs + else: # only just return the prompt + delta_text = res.prompt + token_ids = res.prompt_token_ids + if top_logprobs: + top_logprobs = res.prompt_logprobs + has_echoed[i] = True + if request.logprobs is not None: + logprobs = self._create_logprobs( + token_ids=token_ids, + top_logprobs=top_logprobs, + num_output_top_logprobs=request.logprobs, + initial_text_offset=offsets, + ) + else: + logprobs = None + previous_texts[i] = output.text + previous_num_tokens[i] = len(output.token_ids) + finish_reason = output.finish_reason + response_json = create_stream_response_json( + index=i, + text=delta_text, + logprobs=logprobs, + finish_reason=finish_reason, + ) + yield f"data: {response_json}\n\n" + if output.finish_reason is not None: + logprobs = (LogProbs() + if request.logprobs is not None else None) + prompt_tokens = len(res.prompt_token_ids) + completion_tokens = len(output.token_ids) + final_usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + response_json = create_stream_response_json( + index=i, + text="", + logprobs=logprobs, + finish_reason=output.finish_reason, + usage=final_usage, + ) + yield f"data: {response_json}\n\n" + yield "data: [DONE]\n\n" + + # Streaming response + if stream: + return completion_stream_generator() + + # Non-streaming response + final_res: RequestOutput = None + async for res in result_generator: + if await raw_request.is_disconnected(): + # Abort the request if the client disconnects. + await self.engine.abort(request_id) + return self.create_error_response("Client disconnected") + final_res = res + assert final_res is not None + choices = [] + prompt_token_ids = final_res.prompt_token_ids + prompt_logprobs = final_res.prompt_logprobs + prompt_text = final_res.prompt + for output in final_res.outputs: + if request.logprobs is not None: + if not echo_without_generation: + token_ids = output.token_ids + top_logprobs = output.logprobs + if request.echo: + token_ids = prompt_token_ids + token_ids + top_logprobs = prompt_logprobs + top_logprobs + else: + token_ids = prompt_token_ids + top_logprobs = prompt_logprobs + logprobs = self._create_logprobs( + token_ids=token_ids, + top_logprobs=top_logprobs, + num_output_top_logprobs=request.logprobs, + ) + else: + logprobs = None + if not echo_without_generation: + output_text = output.text + if request.echo: + output_text = prompt_text + output_text + else: + output_text = prompt_text + choice_data = CompletionResponseChoice( + index=output.index, + text=output_text, + logprobs=logprobs, + finish_reason=output.finish_reason, + ) + choices.append(choice_data) + + num_prompt_tokens = len(final_res.prompt_token_ids) + num_generated_tokens = sum( + len(output.token_ids) for output in final_res.outputs) + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + ) + response = CompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=usage, + ) + + if request.stream: + # When user requests streaming but we don't stream, we still need to + # return a streaming response with a single event. + response_json = response.json(ensure_ascii=False) + + async def fake_stream_generator() -> AsyncGenerator[str, None]: + yield f"data: {response_json}\n\n" + yield "data: [DONE]\n\n" + + return fake_stream_generator() + + return response + + def _create_logprobs(self, + token_ids: List[int], + top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None, + num_output_top_logprobs: Optional[int] = None, + initial_text_offset: int = 0, + ) -> LogProbs: + """Create OpenAI-style logprobs.""" + logprobs = LogProbs() + last_token_len = 0 + if num_output_top_logprobs: + logprobs.top_logprobs = [] + for i, token_id in enumerate(token_ids): + step_top_logprobs = top_logprobs[i] + if step_top_logprobs is not None: + token_logprob = step_top_logprobs[token_id] + else: + token_logprob = None + token = self.tokenizer.convert_ids_to_tokens(token_id) + logprobs.tokens.append(token) + logprobs.token_logprobs.append(token_logprob) + if len(logprobs.text_offset) == 0: + logprobs.text_offset.append(initial_text_offset) + else: + logprobs.text_offset.append(logprobs.text_offset[-1] + + last_token_len) + last_token_len = len(token) + + if num_output_top_logprobs: + logprobs.top_logprobs.append({ + self.tokenizer.convert_ids_to_tokens(i): p + for i, p in step_top_logprobs.items() + } if step_top_logprobs else None) + return logprobs + + def create_error_response(self, message: str, + type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse: + return ErrorResponse(message=message, type=type, code=status_code.value) + + async def _check_model(self, request) -> Optional[ErrorResponse]: + print("Modèle : %s" % request.model) + if len(request.model) == 0 or request.model == "gpt-3.5-turbo": # A virer... + return + if request.model == self.served_model: + return + return self.create_error_response(message=f"The model `{request.model}` does not exist.", + type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND) + + async def _check_length(self, + request: Union[ChatCompletionRequest, CompletionRequest], + prompt: Optional[str] = None, + prompt_ids: Optional[List[int]] = None + ) -> Tuple[List[int], Optional[ErrorResponse]]: + assert (not (prompt is None and prompt_ids is None) + and not (prompt is not None and prompt_ids is not None) + ), "Either prompt or prompt_ids should be provided." + input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(prompt).input_ids + token_num = len(input_ids) + + if request.max_tokens is None: + request.max_tokens = self.max_model_len - token_num + if token_num + request.max_tokens > self.max_model_len: + return input_ids, self.create_error_response( + f"This model's maximum context length is {self.max_model_len} tokens. " + f"However, you requested {request.max_tokens + token_num} tokens " + f"({token_num} in the messages, " + f"{request.max_tokens} in the completion). " + f"Please reduce the length of the messages or completion.", + ) + else: + return input_ids, None + + def _load_chat_template(self, chat_template): + if chat_template is not None: + try: + with open(chat_template, "r") as f: + self.tokenizer.chat_template = f.read() + except OSError: + # If opening a file fails, set chat template to be args to + # ensure we decode so our escape are interpreted correctly + self.tokenizer.chat_template = codecs.decode(chat_template, "unicode_escape") + + logger.info( + f"Using supplied chat template:\n{self.tokenizer.chat_template}") + elif self.tokenizer.chat_template is not None: + logger.info(f"Using default chat template:\n{self.tokenizer.chat_template}") + else: + logger.warning("No chat template provided. Chat API will not work.") diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 30a8036a63fc..b5710eef4ad5 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -100,7 +100,7 @@ def __init__( temperature: float = 1.0, top_p: float = 1.0, top_k: int = -1, - min_p: int = 0.0, + min_p: float = 0.0, use_beam_search: bool = False, length_penalty: float = 1.0, early_stopping: Union[bool, str] = False, From d2c6fe586028bfbc5dd260b1864cfa7801c41cfc Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 01:36:49 +0100 Subject: [PATCH 03/16] OpenAI compatible functions calling --- vllm/entrypoints/openai/api_server.py | 10 +- vllm/entrypoints/openai/protocol.py | 42 +++- vllm/entrypoints/openai/serving.py | 334 +++++++++++++++++++++----- 3 files changed, 322 insertions(+), 64 deletions(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 3252005f491c..ce15cd809f76 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -15,7 +15,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine from vllm.engine.metrics import add_global_metrics_labels from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse -from vllm.entrypoints.openai.serving import OpenAIServing +from vllm.entrypoints.openai.serving import OpenAIServing, OpenAIToolsPrompter from vllm.logger import init_logger TIMEOUT_KEEP_ALIVE = 5 # seconds @@ -56,6 +56,10 @@ def parse_args(): help="The file path to the chat template, " "or the template in single-line form " "for the specified model") + parser.add_argument("--enable-api-tools", + action="store_true", + help="Enable OpenAI-like tools API " + "(only function calls are currently supported)") parser.add_argument("--response-role", type=str, default="assistant", @@ -130,8 +134,10 @@ async def create_completion(request: CompletionRequest, raw_request: Request): engine_args = AsyncEngineArgs.from_cli_args(args) engine = AsyncLLMEngine.from_engine_args(engine_args) + enable_tools = True - openai_serving = OpenAIServing(engine, served_model, args.response_role, args.chat_template) + openai_tools_prompter = OpenAIToolsPrompter() if args.enable_api_tools else None + openai_serving = OpenAIServing(engine, served_model, args.response_role, args.chat_template, openai_tools_prompter) # Register labels for metrics add_global_metrics_labels(model_name=engine_args.model) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 7a86a19c4bf8..039ac728bb82 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -1,7 +1,7 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py import time -from typing import Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, Field @@ -52,6 +52,18 @@ class UsageInfo(BaseModel): completion_tokens: Optional[int] = 0 +class ChatCompletionFunctionDef(BaseModel): + name: str + description: str + parameters: Optional[Any] = None + # See : https://json-schema.org/understanding-json-schema/reference/object + + +class ChatCompletionToolParam(BaseModel): + type: str = "function" + function: ChatCompletionFunctionDef = None + + class ChatCompletionRequest(BaseModel): model: str messages: Union[str, List[Dict[str, str]]] @@ -77,6 +89,8 @@ class ChatCompletionRequest(BaseModel): echo: Optional[bool] = False repetition_penalty: Optional[float] = 1.0 min_p: Optional[float] = 0.0 + tools: Optional[List[ChatCompletionToolParam]] = None + tool_choice: Optional[str] = None class CompletionRequest(BaseModel): @@ -147,15 +161,27 @@ class CompletionStreamResponse(BaseModel): usage: Optional[UsageInfo] +class FunctionCall(BaseModel): + name: str + arguments: str + + +class ToolCallsMessage(BaseModel): + id: str + type: str + function: FunctionCall + + class ChatMessage(BaseModel): role: str - content: str + content: Optional[str] = None + tool_calls: Optional[List[ToolCallsMessage]] = None class ChatCompletionResponseChoice(BaseModel): index: int message: ChatMessage - finish_reason: Optional[Literal["stop", "length"]] = None + finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None class ChatCompletionResponse(BaseModel): @@ -167,15 +193,23 @@ class ChatCompletionResponse(BaseModel): usage: UsageInfo +class ToolCallsDelta(BaseModel): + index: int + id: str + type: str + function: FunctionCall + + class DeltaMessage(BaseModel): role: Optional[str] = None content: Optional[str] = None + tool_calls: Optional[ToolCallsDelta] = None class ChatCompletionResponseStreamChoice(BaseModel): index: int delta: DeltaMessage - finish_reason: Optional[Literal["stop", "length"]] = None + finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None class ChatCompletionStreamResponse(BaseModel): diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index 5104ffe04979..2837aeae30a6 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -1,4 +1,5 @@ import time +import json import asyncio import codecs from http import HTTPStatus @@ -11,8 +12,9 @@ CompletionResponseStreamChoice, CompletionStreamResponse, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse, - LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo) + ChatCompletionStreamResponse, ChatMessage, DeltaMessage, + LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, + ChatCompletionToolParam, ToolCallsDelta, ToolCallsMessage, FunctionCall,ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer @@ -20,11 +22,123 @@ logger = init_logger(__name__) +class OpenAIToolsPrompter: + """ + https://platform.openai.com/docs/assistants/tools + """ + def __init__(self): + pass + + @classmethod + def func_call_token_pre(cls) -> str: + return "!" + + @classmethod + def func_call_token_size(cls) -> int: + return 15 + + @classmethod + def func_call_token(cls) -> str: + return "!function_call:" + + def inject_prompt(self, request: ChatCompletionRequest): + """ Tested with : + https://huggingface.co/mlabonne/NeuralHermes-2.5-Mistral-7B/discussions/3 """ + if request.tool_choice is not None and request.tools is not None: + if request.tool_choice == "auto": + tools_list: [ChatCompletionToolParam] = request.tools + if len(tools_list): + text_inject = ( "Your task is to call a function when needed. " + "You will be provided with a list of functions. " + "\n\nAvailable functions:\n") + for tool in tools_list: + if tool.type == "function": + text_inject += "\n" + tool.function.name + if tool.function.description is not None: + text_inject += " - " + tool.function.description + if tool.function.parameters is not None: + schema = json.dumps(tool.function.parameters, indent=4) + text_inject += f"```\njsonschema\n{schema}\n```" + text_inject += (f"\nTo call a function, the response must start by" + f"\"{self.func_call_token()} followed by a json like this: " + f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" + "If you cannot call a function due to lack of information, " + "do not make a function call and ask the user for additional details." + f"If you can call a function, you don't explain anything, just do the call. " + f"You only call functions when it's needed and when the description matches with the user input. " + f"After a function call, the response must terminate immediately.\n\n") + if isinstance(request.messages, str): + request.messages = text_inject + request.messages + elif isinstance(request.messages, List): + if len(request.messages) >= 1: + request.messages[0]["content"] = text_inject + request.messages[0]["content"] + +class PromptCapture: + def __init__(self): + self.content: str = "" + self.ignore = False + self.is_function_call = False + self.calls_list: List[{}] = None + + def make_calls_list(self, prompter: OpenAIToolsPrompter): + calls_list = self.content.split(prompter.func_call_token()) + self.calls_list = [] + for v_call in calls_list: + print(v_call) + if len(v_call): + try: + call_dict = json.loads(v_call) + if "call" in call_dict: + self.calls_list.append(call_dict) + except json.decoder.JSONDecodeError as exc: + # Simply ignore invalid functions calls... + pass + + def num_calls(self): + return len(self.calls_list) if self.calls_list is not None else 0 + + def calls_validation(self, tools_list: [str]) -> bool: + """ Validate function / tool calls by searching name in the tools defined in the request. """ + if self.calls_list is not None: + for ic in range(len(self.calls_list)): + if self.calls_list[ic]["call"] in tools_list: + pass + else: + return False + return True + return False + + def to_ToolCallsMessage(self, func_id: int) -> Union[ToolCallsMessage, None]: + if self.calls_list is not None: + call = self.calls_list[func_id] + arguments = call["params"] if "params" in call else None + function_call = FunctionCall(name=call["call"], + arguments=json.dumps(arguments)) + return ToolCallsMessage(id="call_" + call["call"], + type="function", + function=function_call) + return None + + def to_ToolCallsDelta(self, index: int, func_id: int) -> Union[ToolCallsDelta, None]: + mesg = self.to_ToolCallsMessage(func_id) + if mesg is not None: + return ToolCallsDelta(index=index, + id=mesg.id, + type=mesg.type, + function=mesg.function) + return None class OpenAIServing: - def __init__(self, engine: AsyncLLMEngine, served_mode: str, response_role: str, chat_template=None): + def __init__(self, + engine: AsyncLLMEngine, + served_model: str, + response_role: str, + chat_template=None, + openai_tools_prompter: OpenAIToolsPrompter=None + ): self.engine = engine - self.served_model = served_mode + self.openai_tools_prompter = openai_tools_prompter + self.served_model = served_model self.response_role = response_role engine_model_config = asyncio.run(engine.get_model_config()) @@ -45,14 +159,16 @@ async def show_available_models(self) -> ModelList: permission=[ModelPermission()]) ]) - async def create_chat_completion(self, request: ChatCompletionRequest, raw_request: Request) -> Union[ErrorResponse, AsyncGenerator[str, None], ChatCompletionResponse]: + async def create_chat_completion(self, + request: ChatCompletionRequest, + raw_request: Request + ) -> Union[ErrorResponse, AsyncGenerator[str, None], ChatCompletionResponse]: """Completion API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/chat/create for the API specification. This API mimics the OpenAI ChatCompletion API. NOTE: Currently we do not support the following features: - - function_call (Users should implement this by themselves) - logit_bias (to be supported by vLLM engine) """ error_check_ret = await self._check_model(request) @@ -63,6 +179,9 @@ async def create_chat_completion(self, request: ChatCompletionRequest, raw_reque # TODO: support logit_bias in vLLM engine. return self.create_error_response("logit_bias is not currently supported") + if self.openai_tools_prompter is not None: + self.openai_tools_prompter.inject_prompt(request) + try: prompt = self.tokenizer.apply_chat_template( conversation=request.messages, @@ -160,6 +279,12 @@ async def chat_completion_stream_generator(self, data = chunk.json(exclude_unset=True, ensure_ascii=False) yield f"data: {data}\n\n" + tools_capture_texts = None + tools_list = None + if self.openai_tools_prompter is not None and request.tools is not None: + tools_capture_texts = [PromptCapture()] * request.n + tools_list: [str] = [tool.function.name for tool in request.tools] + # Send response for each token for each request.n (index) previous_texts = [""] * request.n previous_num_tokens = [0] * request.n @@ -172,46 +297,101 @@ async def chat_completion_stream_generator(self, if finish_reason_sent[i]: continue - if output.finish_reason is None: - # Send token-by-token response for each request.n - delta_text = output.text[len(previous_texts[i]):] - previous_texts[i] = output.text - previous_num_tokens[i] = len(output.token_ids) - choice_data = ChatCompletionResponseStreamChoice( - index=i, - delta=DeltaMessage(content=delta_text), - finish_reason=None) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - data = chunk.json(exclude_unset=True, ensure_ascii=False) - yield f"data: {data}\n\n" + # Manage tools calling + if self.openai_tools_prompter is not None and \ + request.tools is not None and \ + not tools_capture_texts[i].ignore: + current_capture = tools_capture_texts[i] + if (len(current_capture.content) == 0) or \ + current_capture.is_function_call or \ + current_capture.content.startswith(OpenAIToolsPrompter.func_call_token_pre()): + current_capture.content += output.text[len(current_capture.content):] + if len(current_capture.content) >= OpenAIToolsPrompter.func_call_token_size() and not current_capture.is_function_call: + if current_capture.content.startswith(OpenAIToolsPrompter.func_call_token()): + current_capture.is_function_call = True + if current_capture.is_function_call: + if output.finish_reason is None: + pass + elif output.finish_reason == "stop": # Send tools call + current_capture.make_calls_list(self.openai_tools_prompter) + if not current_capture.calls_validation(tools_list=tools_list): + current_capture.ignore = True + continue + calls_count = current_capture.num_calls() + for call_id in range(calls_count): + prompt_tokens = len(res.prompt_token_ids) + if call_id == (calls_count-1): + final_usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=len(output.token_ids), + total_tokens=prompt_tokens + len(output.token_ids), + ) + else: + final_usage = None + tool_call = current_capture.to_ToolCallsDelta(index=call_id, func_id=call_id) + choice_data = ChatCompletionResponseStreamChoice( + index=i, delta=DeltaMessage(content=None, tool_calls=tool_call), finish_reason="tool_calls") + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if final_usage is not None: + chunk.usage = final_usage + data = chunk.json(exclude_unset=True, + exclude_none=True, + ensure_ascii=False) + yield f"data: {data}\n\n" + finish_reason_sent[i] = True + current_capture.ignore = True + else: + current_capture.ignore = True + else: + pass + else: + current_capture.ignore = True else: - # Send the finish response for each request.n only once - prompt_tokens = len(res.prompt_token_ids) - final_usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=previous_num_tokens[i], - total_tokens=prompt_tokens + previous_num_tokens[i], - ) - choice_data = ChatCompletionResponseStreamChoice( - index=i, delta=[], finish_reason=output.finish_reason) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - if final_usage is not None: - chunk.usage = final_usage - data = chunk.json(exclude_unset=True, - exclude_none=True, - ensure_ascii=False) - yield f"data: {data}\n\n" - finish_reason_sent[i] = True + if output.finish_reason is None: + # Send token-by-token response for each request.n + delta_text = output.text[len(previous_texts[i]):] + previous_texts[i] = output.text + previous_num_tokens[i] = len(output.token_ids) + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage(content=delta_text), + finish_reason=None) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + data = chunk.json(exclude_unset=True, ensure_ascii=False) + yield f"data: {data}\n\n" + else: + # Send the finish response for each request.n only once + prompt_tokens = len(res.prompt_token_ids) + final_usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=previous_num_tokens[i], + total_tokens=prompt_tokens + previous_num_tokens[i], + ) + choice_data = ChatCompletionResponseStreamChoice( + index=i, delta=[], finish_reason=output.finish_reason) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if final_usage is not None: + chunk.usage = final_usage + data = chunk.json(exclude_unset=True, + exclude_none=True, + ensure_ascii=False) + yield f"data: {data}\n\n" + finish_reason_sent[i] = True # Send the final done message after all response.n are finished yield "data: [DONE]\n\n" @@ -219,7 +399,8 @@ async def chat_completion_full_generator(self, request: ChatCompletionRequest, raw_request: Request, result_generator: AsyncIterator[RequestOutput], - request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]: + request_id: str + ) -> Union[ErrorResponse, ChatCompletionResponse]: model_name = request.model created_time = int(time.monotonic()) @@ -235,12 +416,47 @@ async def chat_completion_full_generator(self, choices = [] role = self.get_chat_request_role(request) for output in final_res.outputs: - choice_data = ChatCompletionResponseChoice( - index=output.index, - message=ChatMessage(role=role, content=output.text), - finish_reason=output.finish_reason, - ) - choices.append(choice_data) + # Manage tools calling + if self.openai_tools_prompter is not None and \ + request.tools is not None: + tools_list: [str] = [tool.function.name for tool in request.tools] + current_capture = PromptCapture() + current_capture.content = output.text + if output.text.startswith(OpenAIToolsPrompter.func_call_token_pre()): + if output.finish_reason == "stop": # Send tools call + current_capture.make_calls_list(self.openai_tools_prompter) + if not current_capture.calls_validation(tools_list=tools_list): + current_capture.ignore = True + else: + calls_count = current_capture.num_calls() + tool_calls = [] + for call_id in range(calls_count): + tool_calls.append(current_capture.to_ToolCallsMessage(func_id=call_id)) + message = ChatMessage( + role=role, + content=None, + tool_calls=tool_calls) + choice_data = ChatCompletionResponseChoice( + index=output.index, message=message, finish_reason="tool_calls") + choices.append(choice_data) + else: + current_capture.ignore = True + else: + current_capture.ignore = True + if current_capture.ignore: + choice_data = ChatCompletionResponseChoice( + index=output.index, + message=ChatMessage(role=role, content=output.text), + finish_reason=output.finish_reason, + ) + choices.append(choice_data) + else: + choice_data = ChatCompletionResponseChoice( + index=output.index, + message=ChatMessage(role=role, content=output.text), + finish_reason=output.finish_reason, + ) + choices.append(choice_data) if request.echo: last_msg_content = "" @@ -570,18 +786,20 @@ def _create_logprobs(self, return logprobs def create_error_response(self, message: str, - type: str = "BadRequestError", + err_type: str = "BadRequestError", status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse: - return ErrorResponse(message=message, type=type, code=status_code.value) + return ErrorResponse(message=message, type=err_type, code=status_code.value) + + async def create_error_generator(self, message: str, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> AsyncGenerator[str, None]: + yield ErrorResponse(message=message, type=err_type, code=status_code.value).json(ensure_ascii=False) async def _check_model(self, request) -> Optional[ErrorResponse]: - print("Modèle : %s" % request.model) - if len(request.model) == 0 or request.model == "gpt-3.5-turbo": # A virer... - return if request.model == self.served_model: return return self.create_error_response(message=f"The model `{request.model}` does not exist.", - type="NotFoundError", + err_type="NotFoundError", status_code=HTTPStatus.NOT_FOUND) async def _check_length(self, From 588785bad76535140546333cebc866ed2560efbd Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 03:11:26 +0100 Subject: [PATCH 04/16] typo --- vllm/entrypoints/openai/serving.py | 64 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index 2837aeae30a6..9010fa92537d 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -44,34 +44,32 @@ def func_call_token(cls) -> str: def inject_prompt(self, request: ChatCompletionRequest): """ Tested with : https://huggingface.co/mlabonne/NeuralHermes-2.5-Mistral-7B/discussions/3 """ - if request.tool_choice is not None and request.tools is not None: - if request.tool_choice == "auto": - tools_list: [ChatCompletionToolParam] = request.tools - if len(tools_list): - text_inject = ( "Your task is to call a function when needed. " - "You will be provided with a list of functions. " - "\n\nAvailable functions:\n") - for tool in tools_list: - if tool.type == "function": - text_inject += "\n" + tool.function.name - if tool.function.description is not None: - text_inject += " - " + tool.function.description - if tool.function.parameters is not None: - schema = json.dumps(tool.function.parameters, indent=4) - text_inject += f"```\njsonschema\n{schema}\n```" - text_inject += (f"\nTo call a function, the response must start by" - f"\"{self.func_call_token()} followed by a json like this: " - f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" - "If you cannot call a function due to lack of information, " - "do not make a function call and ask the user for additional details." - f"If you can call a function, you don't explain anything, just do the call. " - f"You only call functions when it's needed and when the description matches with the user input. " - f"After a function call, the response must terminate immediately.\n\n") - if isinstance(request.messages, str): - request.messages = text_inject + request.messages - elif isinstance(request.messages, List): - if len(request.messages) >= 1: - request.messages[0]["content"] = text_inject + request.messages[0]["content"] + if request.tool_choice is not None and request.tools is not None and request.tool_choice == "auto": + tools_list: [ChatCompletionToolParam] = request.tools + if len(tools_list): + text_inject = ( "Your task is to call a function when needed. " + "You will be provided with a list of functions. " + "\n\nAvailable functions:\n") + for tool in tools_list: + if tool.type == "function": + text_inject += "\n" + tool.function.name + if tool.function.description is not None: + text_inject += " - " + tool.function.description + if tool.function.parameters is not None: + schema = json.dumps(tool.function.parameters, indent=4) + text_inject += f"```\njsonschema\n{schema}\n```" + text_inject += (f"\nTo call a function, the response must start by" + f"\"{self.func_call_token()} followed by a json like this: " + f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" + "If you cannot call a function due to lack of information, " + "do not make a function call and ask the user for additional details." + f"If you can call a function, you don't explain anything, just do the call. " + f"You only call functions when it's needed and when the description matches with the user input. " + f"After a function call, the response must terminate immediately.\n\n") + if isinstance(request.messages, str): + request.messages = text_inject + request.messages + elif isinstance(request.messages, List) and len(request.messages) >= 1: + request.messages[0]["content"] = text_inject + request.messages[0]["content"] class PromptCapture: def __init__(self): @@ -90,7 +88,7 @@ def make_calls_list(self, prompter: OpenAIToolsPrompter): call_dict = json.loads(v_call) if "call" in call_dict: self.calls_list.append(call_dict) - except json.decoder.JSONDecodeError as exc: + except json.decoder.JSONDecodeError: # Simply ignore invalid functions calls... pass @@ -306,9 +304,10 @@ async def chat_completion_stream_generator(self, current_capture.is_function_call or \ current_capture.content.startswith(OpenAIToolsPrompter.func_call_token_pre()): current_capture.content += output.text[len(current_capture.content):] - if len(current_capture.content) >= OpenAIToolsPrompter.func_call_token_size() and not current_capture.is_function_call: - if current_capture.content.startswith(OpenAIToolsPrompter.func_call_token()): - current_capture.is_function_call = True + if len(current_capture.content) >= OpenAIToolsPrompter.func_call_token_size() and \ + not current_capture.is_function_call and \ + current_capture.content.startswith(OpenAIToolsPrompter.func_call_token()): + current_capture.is_function_call = True if current_capture.is_function_call: if output.finish_reason is None: pass @@ -796,6 +795,7 @@ async def create_error_generator(self, message: str, yield ErrorResponse(message=message, type=err_type, code=status_code.value).json(ensure_ascii=False) async def _check_model(self, request) -> Optional[ErrorResponse]: + return if request.model == self.served_model: return return self.create_error_response(message=f"The model `{request.model}` does not exist.", From 480845c4a38e2443358e407df1d87967779341c7 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 10:41:48 +0100 Subject: [PATCH 05/16] format --- vllm/entrypoints/openai/serving.py | 294 ++++++++++++++++++----------- 1 file changed, 180 insertions(+), 114 deletions(-) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index 9010fa92537d..bc3b4acdd8f9 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -12,9 +12,9 @@ CompletionResponseStreamChoice, CompletionStreamResponse, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, ChatMessage, DeltaMessage, - LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, - ChatCompletionToolParam, ToolCallsDelta, ToolCallsMessage, FunctionCall,ErrorResponse) + ChatCompletionStreamResponse, ChatMessage, DeltaMessage, LogProbs, + ModelCard, ModelList, ModelPermission, UsageInfo, ChatCompletionToolParam, + ToolCallsDelta, ToolCallsMessage, FunctionCall, ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer @@ -22,10 +22,12 @@ logger = init_logger(__name__) + class OpenAIToolsPrompter: """ https://platform.openai.com/docs/assistants/tools """ + def __init__(self): pass @@ -47,31 +49,40 @@ def inject_prompt(self, request: ChatCompletionRequest): if request.tool_choice is not None and request.tools is not None and request.tool_choice == "auto": tools_list: [ChatCompletionToolParam] = request.tools if len(tools_list): - text_inject = ( "Your task is to call a function when needed. " - "You will be provided with a list of functions. " - "\n\nAvailable functions:\n") + text_inject = ( + "Your task is to call a function when needed. " + "You will be provided with a list of functions. " + "\n\nAvailable functions:\n") for tool in tools_list: if tool.type == "function": text_inject += "\n" + tool.function.name if tool.function.description is not None: text_inject += " - " + tool.function.description if tool.function.parameters is not None: - schema = json.dumps(tool.function.parameters, indent=4) + schema = json.dumps(tool.function.parameters, + indent=4) text_inject += f"```\njsonschema\n{schema}\n```" - text_inject += (f"\nTo call a function, the response must start by" - f"\"{self.func_call_token()} followed by a json like this: " - f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" - "If you cannot call a function due to lack of information, " - "do not make a function call and ask the user for additional details." - f"If you can call a function, you don't explain anything, just do the call. " - f"You only call functions when it's needed and when the description matches with the user input. " - f"After a function call, the response must terminate immediately.\n\n") + text_inject += ( + f"\nTo call a function, the response must start by" + f"\"{self.func_call_token()} followed by a json like this: " + f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" + "If you cannot call a function due to lack of information, " + "do not make a function call and ask the user for additional details." + f"If you can call a function, you don't explain anything, just do the call. " + f"You only call functions when it's needed and when the description matches with the user input. " + f"After a function call, the response must terminate immediately.\n\n" + ) if isinstance(request.messages, str): request.messages = text_inject + request.messages - elif isinstance(request.messages, List) and len(request.messages) >= 1: - request.messages[0]["content"] = text_inject + request.messages[0]["content"] + elif isinstance(request.messages, + List) and len(request.messages) >= 1: + request.messages[0][ + "content"] = text_inject + request.messages[0][ + "content"] + class PromptCapture: + def __init__(self): self.content: str = "" self.ignore = False @@ -106,7 +117,8 @@ def calls_validation(self, tools_list: [str]) -> bool: return True return False - def to_ToolCallsMessage(self, func_id: int) -> Union[ToolCallsMessage, None]: + def to_ToolCallsMessage(self, + func_id: int) -> Union[ToolCallsMessage, None]: if self.calls_list is not None: call = self.calls_list[func_id] arguments = call["params"] if "params" in call else None @@ -117,7 +129,8 @@ def to_ToolCallsMessage(self, func_id: int) -> Union[ToolCallsMessage, None]: function=function_call) return None - def to_ToolCallsDelta(self, index: int, func_id: int) -> Union[ToolCallsDelta, None]: + def to_ToolCallsDelta(self, index: int, + func_id: int) -> Union[ToolCallsDelta, None]: mesg = self.to_ToolCallsMessage(func_id) if mesg is not None: return ToolCallsDelta(index=index, @@ -126,14 +139,15 @@ def to_ToolCallsDelta(self, index: int, func_id: int) -> Union[ToolCallsDelta, N function=mesg.function) return None + class OpenAIServing: + def __init__(self, engine: AsyncLLMEngine, served_model: str, response_role: str, chat_template=None, - openai_tools_prompter: OpenAIToolsPrompter=None - ): + openai_tools_prompter: OpenAIToolsPrompter = None): self.engine = engine self.openai_tools_prompter = openai_tools_prompter self.served_model = served_model @@ -157,10 +171,10 @@ async def show_available_models(self) -> ModelList: permission=[ModelPermission()]) ]) - async def create_chat_completion(self, - request: ChatCompletionRequest, - raw_request: Request - ) -> Union[ErrorResponse, AsyncGenerator[str, None], ChatCompletionResponse]: + async def create_chat_completion( + self, request: ChatCompletionRequest, raw_request: Request + ) -> Union[ErrorResponse, AsyncGenerator[str, None], + ChatCompletionResponse]: """Completion API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/chat/create @@ -175,7 +189,8 @@ async def create_chat_completion(self, if request.logit_bias is not None and len(request.logit_bias) > 0: # TODO: support logit_bias in vLLM engine. - return self.create_error_response("logit_bias is not currently supported") + return self.create_error_response( + "logit_bias is not currently supported") if self.openai_tools_prompter is not None: self.openai_tools_prompter.inject_prompt(request) @@ -186,10 +201,12 @@ async def create_chat_completion(self, tokenize=False, add_generation_prompt=request.add_generation_prompt) except Exception as e: - logger.error(f"Error in applying chat template from request: {str(e)}") + logger.error( + f"Error in applying chat template from request: {str(e)}") return self.create_error_response(str(e)) - token_ids, error_check_ret = await self._check_length(request, prompt=prompt) + token_ids, error_check_ret = await self._check_length(request, + prompt=prompt) if error_check_ret is not None: return error_check_ret @@ -217,13 +234,16 @@ async def create_chat_completion(self, return self.create_error_response(str(e)) request_id = f"cmpl-{random_uuid()}" - result_generator = self.engine.generate(prompt, sampling_params, request_id, token_ids) + result_generator = self.engine.generate(prompt, sampling_params, + request_id, token_ids) # Streaming response if request.stream: - return self.chat_completion_stream_generator(request, result_generator, request_id) + return self.chat_completion_stream_generator( + request, result_generator, request_id) else: - return await self.chat_completion_full_generator(request, raw_request, result_generator, request_id) + return await self.chat_completion_full_generator( + request, raw_request, result_generator, request_id) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: @@ -231,10 +251,9 @@ def get_chat_request_role(self, request: ChatCompletionRequest) -> str: else: return request.messages[-1]["role"] - async def chat_completion_stream_generator(self, - request: ChatCompletionRequest, - result_generator: AsyncIterator[RequestOutput], - request_id: str + async def chat_completion_stream_generator( + self, request: ChatCompletionRequest, + result_generator: AsyncIterator[RequestOutput], request_id: str ) -> Union[ErrorResponse, AsyncGenerator[str, None]]: model_name = request.model @@ -303,7 +322,8 @@ async def chat_completion_stream_generator(self, if (len(current_capture.content) == 0) or \ current_capture.is_function_call or \ current_capture.content.startswith(OpenAIToolsPrompter.func_call_token_pre()): - current_capture.content += output.text[len(current_capture.content):] + current_capture.content += output.text[ + len(current_capture.content):] if len(current_capture.content) >= OpenAIToolsPrompter.func_call_token_size() and \ not current_capture.is_function_call and \ current_capture.content.startswith(OpenAIToolsPrompter.func_call_token()): @@ -312,24 +332,33 @@ async def chat_completion_stream_generator(self, if output.finish_reason is None: pass elif output.finish_reason == "stop": # Send tools call - current_capture.make_calls_list(self.openai_tools_prompter) - if not current_capture.calls_validation(tools_list=tools_list): + current_capture.make_calls_list( + self.openai_tools_prompter) + if not current_capture.calls_validation( + tools_list=tools_list): current_capture.ignore = True continue calls_count = current_capture.num_calls() for call_id in range(calls_count): prompt_tokens = len(res.prompt_token_ids) - if call_id == (calls_count-1): + if call_id == (calls_count - 1): final_usage = UsageInfo( prompt_tokens=prompt_tokens, - completion_tokens=len(output.token_ids), - total_tokens=prompt_tokens + len(output.token_ids), + completion_tokens=len( + output.token_ids), + total_tokens=prompt_tokens + + len(output.token_ids), ) else: final_usage = None - tool_call = current_capture.to_ToolCallsDelta(index=call_id, func_id=call_id) + tool_call = current_capture.to_ToolCallsDelta( + index=call_id, func_id=call_id) choice_data = ChatCompletionResponseStreamChoice( - index=i, delta=DeltaMessage(content=None, tool_calls=tool_call), finish_reason="tool_calls") + index=i, + delta=DeltaMessage( + content=None, + tool_calls=tool_call), + finish_reason="tool_calls") chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, @@ -366,7 +395,8 @@ async def chat_completion_stream_generator(self, created=created_time, choices=[choice_data], model=model_name) - data = chunk.json(exclude_unset=True, ensure_ascii=False) + data = chunk.json(exclude_unset=True, + ensure_ascii=False) yield f"data: {data}\n\n" else: # Send the finish response for each request.n only once @@ -374,10 +404,13 @@ async def chat_completion_stream_generator(self, final_usage = UsageInfo( prompt_tokens=prompt_tokens, completion_tokens=previous_num_tokens[i], - total_tokens=prompt_tokens + previous_num_tokens[i], + total_tokens=prompt_tokens + + previous_num_tokens[i], ) choice_data = ChatCompletionResponseStreamChoice( - index=i, delta=[], finish_reason=output.finish_reason) + index=i, + delta=[], + finish_reason=output.finish_reason) chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, @@ -394,12 +427,10 @@ async def chat_completion_stream_generator(self, # Send the final done message after all response.n are finished yield "data: [DONE]\n\n" - async def chat_completion_full_generator(self, - request: ChatCompletionRequest, - raw_request: Request, - result_generator: AsyncIterator[RequestOutput], - request_id: str - ) -> Union[ErrorResponse, ChatCompletionResponse]: + async def chat_completion_full_generator( + self, request: ChatCompletionRequest, raw_request: Request, + result_generator: AsyncIterator[RequestOutput], + request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]: model_name = request.model created_time = int(time.monotonic()) @@ -418,25 +449,33 @@ async def chat_completion_full_generator(self, # Manage tools calling if self.openai_tools_prompter is not None and \ request.tools is not None: - tools_list: [str] = [tool.function.name for tool in request.tools] + tools_list: [str] = [ + tool.function.name for tool in request.tools + ] current_capture = PromptCapture() current_capture.content = output.text - if output.text.startswith(OpenAIToolsPrompter.func_call_token_pre()): + if output.text.startswith( + OpenAIToolsPrompter.func_call_token_pre()): if output.finish_reason == "stop": # Send tools call - current_capture.make_calls_list(self.openai_tools_prompter) - if not current_capture.calls_validation(tools_list=tools_list): + current_capture.make_calls_list( + self.openai_tools_prompter) + if not current_capture.calls_validation( + tools_list=tools_list): current_capture.ignore = True else: calls_count = current_capture.num_calls() tool_calls = [] for call_id in range(calls_count): - tool_calls.append(current_capture.to_ToolCallsMessage(func_id=call_id)) - message = ChatMessage( - role=role, - content=None, - tool_calls=tool_calls) + tool_calls.append( + current_capture.to_ToolCallsMessage( + func_id=call_id)) + message = ChatMessage(role=role, + content=None, + tool_calls=tool_calls) choice_data = ChatCompletionResponseChoice( - index=output.index, message=message, finish_reason="tool_calls") + index=output.index, + message=message, + finish_reason="tool_calls") choices.append(choice_data) else: current_capture.ignore = True @@ -461,8 +500,8 @@ async def chat_completion_full_generator(self, last_msg_content = "" if request.messages and isinstance( request.messages, list) and request.messages[-1].get( - "content") and request.messages[-1].get( - "role") == role: + "content") and request.messages[-1].get( + "role") == role: last_msg_content = request.messages[-1]["content"] for choice in choices: @@ -486,7 +525,8 @@ async def chat_completion_full_generator(self, ) return response - async def create_completion(self, request: CompletionRequest, raw_request: Request): + async def create_completion(self, request: CompletionRequest, + raw_request: Request): """Completion API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/completions/create @@ -507,11 +547,13 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque if request.suffix is not None: # The language models we currently support do not support suffix. - return self.create_error_response("suffix is not currently supported") + return self.create_error_response( + "suffix is not currently supported") if request.logit_bias is not None and len(request.logit_bias) > 0: # TODO: support logit_bias in vLLM engine. - return self.create_error_response("logit_bias is not currently supported") + return self.create_error_response( + "logit_bias is not currently supported") model_name = request.model request_id = f"cmpl-{random_uuid()}" @@ -519,7 +561,8 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque use_token_ids = False if isinstance(request.prompt, list): if len(request.prompt) == 0: - return self.create_error_response("please provide at least one prompt") + return self.create_error_response( + "please provide at least one prompt") first_element = request.prompt[0] if isinstance(first_element, int): use_token_ids = True @@ -527,16 +570,20 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque elif isinstance(first_element, (str, list)): # TODO: handles multiple prompt case in list[list[int]] if len(request.prompt) > 1: - return self.create_error_response("multiple prompts in a batch is not currently supported") + return self.create_error_response( + "multiple prompts in a batch is not currently supported" + ) use_token_ids = not isinstance(first_element, str) prompt = request.prompt[0] else: prompt = request.prompt if use_token_ids: - _, error_check_ret = await self._check_length(request, prompt_ids=prompt) + _, error_check_ret = await self._check_length(request, + prompt_ids=prompt) else: - token_ids, error_check_ret = await self._check_length(request, prompt=prompt) + token_ids, error_check_ret = await self._check_length( + request, prompt=prompt) if error_check_ret is not None: return error_check_ret @@ -556,7 +603,8 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque stop=request.stop, stop_token_ids=request.stop_token_ids, ignore_eos=request.ignore_eos, - max_tokens=request.max_tokens if not echo_without_generation else 1, + max_tokens=request.max_tokens + if not echo_without_generation else 1, logprobs=request.logprobs, use_beam_search=request.use_beam_search, prompt_logprobs=request.logprobs if request.echo else None, @@ -572,8 +620,8 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque request_id, prompt_token_ids=prompt) else: - result_generator = self.engine.generate(prompt, sampling_params, request_id, - token_ids) + result_generator = self.engine.generate(prompt, sampling_params, + request_id, token_ids) # Similar to the OpenAI API, when n != best_of, we do not stream the # results. In addition, we do not stream the results when use beam search. @@ -582,11 +630,11 @@ async def create_completion(self, request: CompletionRequest, raw_request: Reque and not request.use_beam_search) def create_stream_response_json( - index: int, - text: str, - logprobs: Optional[LogProbs] = None, - finish_reason: Optional[str] = None, - usage: Optional[UsageInfo] = None, + index: int, + text: str, + logprobs: Optional[LogProbs] = None, + finish_reason: Optional[str] = None, + usage: Optional[UsageInfo] = None, ) -> str: choice_data = CompletionResponseStreamChoice( index=index, @@ -602,7 +650,8 @@ def create_stream_response_json( ) if usage is not None: response.usage = usage - response_json = response.json(exclude_unset=True, ensure_ascii=False) + response_json = response.json(exclude_unset=True, + ensure_ascii=False) return response_json @@ -750,12 +799,13 @@ async def fake_stream_generator() -> AsyncGenerator[str, None]: return response - def _create_logprobs(self, - token_ids: List[int], - top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None, - num_output_top_logprobs: Optional[int] = None, - initial_text_offset: int = 0, - ) -> LogProbs: + def _create_logprobs( + self, + token_ids: List[int], + top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None, + num_output_top_logprobs: Optional[int] = None, + initial_text_offset: int = 0, + ) -> LogProbs: """Create OpenAI-style logprobs.""" logprobs = LogProbs() last_token_len = 0 @@ -779,38 +829,50 @@ def _create_logprobs(self, if num_output_top_logprobs: logprobs.top_logprobs.append({ - self.tokenizer.convert_ids_to_tokens(i): p - for i, p in step_top_logprobs.items() - } if step_top_logprobs else None) + self.tokenizer.convert_ids_to_tokens(i): p + for i, p in step_top_logprobs.items() + } if step_top_logprobs else None) return logprobs - def create_error_response(self, message: str, - err_type: str = "BadRequestError", - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse: - return ErrorResponse(message=message, type=err_type, code=status_code.value) - - async def create_error_generator(self, message: str, - err_type: str = "BadRequestError", - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> AsyncGenerator[str, None]: - yield ErrorResponse(message=message, type=err_type, code=status_code.value).json(ensure_ascii=False) + def create_error_response( + self, + message: str, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse: + return ErrorResponse(message=message, + type=err_type, + code=status_code.value) + + async def create_error_generator( + self, + message: str, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST + ) -> AsyncGenerator[str, None]: + yield ErrorResponse(message=message, + type=err_type, + code=status_code.value).json(ensure_ascii=False) async def _check_model(self, request) -> Optional[ErrorResponse]: return if request.model == self.served_model: return - return self.create_error_response(message=f"The model `{request.model}` does not exist.", - err_type="NotFoundError", - status_code=HTTPStatus.NOT_FOUND) - - async def _check_length(self, - request: Union[ChatCompletionRequest, CompletionRequest], - prompt: Optional[str] = None, - prompt_ids: Optional[List[int]] = None - ) -> Tuple[List[int], Optional[ErrorResponse]]: + return self.create_error_response( + message=f"The model `{request.model}` does not exist.", + err_type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND) + + async def _check_length( + self, + request: Union[ChatCompletionRequest, CompletionRequest], + prompt: Optional[str] = None, + prompt_ids: Optional[List[int]] = None + ) -> Tuple[List[int], Optional[ErrorResponse]]: assert (not (prompt is None and prompt_ids is None) and not (prompt is not None and prompt_ids is not None) ), "Either prompt or prompt_ids should be provided." - input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(prompt).input_ids + input_ids = prompt_ids if prompt_ids is not None else self.tokenizer( + prompt).input_ids token_num = len(input_ids) if request.max_tokens is None: @@ -821,8 +883,7 @@ async def _check_length(self, f"However, you requested {request.max_tokens + token_num} tokens " f"({token_num} in the messages, " f"{request.max_tokens} in the completion). " - f"Please reduce the length of the messages or completion.", - ) + f"Please reduce the length of the messages or completion.", ) else: return input_ids, None @@ -834,11 +895,16 @@ def _load_chat_template(self, chat_template): except OSError: # If opening a file fails, set chat template to be args to # ensure we decode so our escape are interpreted correctly - self.tokenizer.chat_template = codecs.decode(chat_template, "unicode_escape") + self.tokenizer.chat_template = codecs.decode( + chat_template, "unicode_escape") logger.info( - f"Using supplied chat template:\n{self.tokenizer.chat_template}") + f"Using supplied chat template:\n{self.tokenizer.chat_template}" + ) elif self.tokenizer.chat_template is not None: - logger.info(f"Using default chat template:\n{self.tokenizer.chat_template}") + logger.info( + f"Using default chat template:\n{self.tokenizer.chat_template}" + ) else: - logger.warning("No chat template provided. Chat API will not work.") + logger.warning( + "No chat template provided. Chat API will not work.") From e7fa1eaa5d61c7af701608056dd255962a231412 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 14:28:27 +0100 Subject: [PATCH 06/16] Fix: Response was not an array in streaming mode, format --- vllm/entrypoints/openai/api_server.py | 23 +++++++++++++++++------ vllm/entrypoints/openai/protocol.py | 2 +- vllm/entrypoints/openai/serving.py | 4 +--- vllm/sampling_params.py | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index ce15cd809f76..0af6ad188c06 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -24,6 +24,7 @@ logger = init_logger(__name__) app = fastapi.FastAPI() + def parse_args(): parser = argparse.ArgumentParser( description="vLLM OpenAI-Compatible RESTful API server.") @@ -86,30 +87,38 @@ def parse_args(): async def validation_exception_handler(_, exc): return openai_serving.create_error_response(message=str(exc)) + @app.get("/health") async def health() -> Response: """Health check.""" return Response(status_code=200) + @app.get("/v1/models") async def show_available_models(): models = await openai_serving.show_available_models() return JSONResponse(content=models.dict()) pass + @app.post("/v1/chat/completions") -async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request): - generator = await openai_serving.create_chat_completion(request, raw_request) +async def create_chat_completion(request: ChatCompletionRequest, + raw_request: Request): + generator = await openai_serving.create_chat_completion( + request, raw_request) if request.stream and not isinstance(generator, ErrorResponse): - return StreamingResponse(content=generator, media_type="text/event-stream") + return StreamingResponse(content=generator, + media_type="text/event-stream") else: return JSONResponse(content=generator.dict()) + @app.post("/v1/completions") async def create_completion(request: CompletionRequest, raw_request: Request): generator = await openai_serving.create_completion(request, raw_request) if request.stream and not isinstance(generator, ErrorResponse): - return StreamingResponse(content=generator, media_type="text/event-stream") + return StreamingResponse(content=generator, + media_type="text/event-stream") else: return JSONResponse(content=generator.dict()) @@ -136,8 +145,10 @@ async def create_completion(request: CompletionRequest, raw_request: Request): engine = AsyncLLMEngine.from_engine_args(engine_args) enable_tools = True - openai_tools_prompter = OpenAIToolsPrompter() if args.enable_api_tools else None - openai_serving = OpenAIServing(engine, served_model, args.response_role, args.chat_template, openai_tools_prompter) + openai_tools_prompter = OpenAIToolsPrompter( + ) if args.enable_api_tools else None + openai_serving = OpenAIServing(engine, served_model, args.response_role, + args.chat_template, openai_tools_prompter) # Register labels for metrics add_global_metrics_labels(model_name=engine_args.model) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 039ac728bb82..3346034c59ea 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -203,7 +203,7 @@ class ToolCallsDelta(BaseModel): class DeltaMessage(BaseModel): role: Optional[str] = None content: Optional[str] = None - tool_calls: Optional[ToolCallsDelta] = None + tool_calls: Optional[List[ToolCallsDelta]] = None class ChatCompletionResponseStreamChoice(BaseModel): diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index bc3b4acdd8f9..aa31d399c048 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -93,7 +93,6 @@ def make_calls_list(self, prompter: OpenAIToolsPrompter): calls_list = self.content.split(prompter.func_call_token()) self.calls_list = [] for v_call in calls_list: - print(v_call) if len(v_call): try: call_dict = json.loads(v_call) @@ -357,7 +356,7 @@ async def chat_completion_stream_generator( index=i, delta=DeltaMessage( content=None, - tool_calls=tool_call), + tool_calls=[tool_call]), finish_reason="tool_calls") chunk = ChatCompletionStreamResponse( id=request_id, @@ -854,7 +853,6 @@ async def create_error_generator( code=status_code.value).json(ensure_ascii=False) async def _check_model(self, request) -> Optional[ErrorResponse]: - return if request.model == self.served_model: return return self.create_error_response( diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index b5710eef4ad5..30a8036a63fc 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -100,7 +100,7 @@ def __init__( temperature: float = 1.0, top_p: float = 1.0, top_k: int = -1, - min_p: float = 0.0, + min_p: int = 0.0, use_beam_search: bool = False, length_penalty: float = 1.0, early_stopping: Union[bool, str] = False, From d2bd2207919be6304e2008b36ec4d0ed57949df8 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 18:36:27 +0100 Subject: [PATCH 07/16] dict from chat messages replaced by types --- vllm/entrypoints/openai/protocol.py | 52 ++++++++++++++++++++++------- vllm/entrypoints/openai/serving.py | 27 ++++++++------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 3346034c59ea..9d28caae193e 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -52,6 +52,17 @@ class UsageInfo(BaseModel): completion_tokens: Optional[int] = 0 +class FunctionCall(BaseModel): + name: str + arguments: str + + +class ToolCallsMessage(BaseModel): + id: str + type: str + function: FunctionCall + + class ChatCompletionFunctionDef(BaseModel): name: str description: str @@ -64,9 +75,37 @@ class ChatCompletionToolParam(BaseModel): function: ChatCompletionFunctionDef = None +class ChatCompletionSystemMessage(BaseModel): + role: str + content: str + name: Optional[str] = None + + +class ChatCompletionUserMessage(BaseModel): + content: Union[str, List[str]] + role: str + name: Optional[str] = None + + +class ChatCompletionAssistantMessage(BaseModel): + content: Optional[str] = None + role: str + name: Optional[str] = None + tool_calls: Optional[List[ToolCallsMessage]] = None + + +class ChatCompletionToolMessage(BaseModel): + content: str + role: str + tool_call_id: str + + class ChatCompletionRequest(BaseModel): model: str - messages: Union[str, List[Dict[str, str]]] + messages: Union[str, List[ChatCompletionSystemMessage], + List[ChatCompletionUserMessage], + List[ChatCompletionAssistantMessage], + List[ChatCompletionToolMessage]] temperature: Optional[float] = 0.7 top_p: Optional[float] = 1.0 n: Optional[int] = 1 @@ -161,17 +200,6 @@ class CompletionStreamResponse(BaseModel): usage: Optional[UsageInfo] -class FunctionCall(BaseModel): - name: str - arguments: str - - -class ToolCallsMessage(BaseModel): - id: str - type: str - function: FunctionCall - - class ChatMessage(BaseModel): role: str content: Optional[str] = None diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index aa31d399c048..e727fa76fc16 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -76,9 +76,8 @@ def inject_prompt(self, request: ChatCompletionRequest): request.messages = text_inject + request.messages elif isinstance(request.messages, List) and len(request.messages) >= 1: - request.messages[0][ - "content"] = text_inject + request.messages[0][ - "content"] + request.messages[ + 0].content = text_inject + request.messages[0].content class PromptCapture: @@ -248,7 +247,7 @@ def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: return self.response_role else: - return request.messages[-1]["role"] + return request.messages[-1].role async def chat_completion_stream_generator( self, request: ChatCompletionRequest, @@ -275,11 +274,11 @@ async def chat_completion_stream_generator( # Send response to echo the input portion of the last message if request.echo: last_msg_content = "" - if request.messages and isinstance( - request.messages, list) and request.messages[-1].get( - "content") and request.messages[-1].get( - "role") == role: - last_msg_content = request.messages[-1]["content"] + if request.messages and \ + isinstance(request.messages, list) and \ + request.messages[-1].content and \ + request.messages[-1].role == role: + last_msg_content = request.messages[-1].content if last_msg_content: for i in range(request.n): choice_data = ChatCompletionResponseStreamChoice( @@ -497,11 +496,11 @@ async def chat_completion_full_generator( if request.echo: last_msg_content = "" - if request.messages and isinstance( - request.messages, list) and request.messages[-1].get( - "content") and request.messages[-1].get( - "role") == role: - last_msg_content = request.messages[-1]["content"] + if request.messages and \ + isinstance(request.messages, list) and \ + request.messages[-1].content and \ + request.messages[-1].role == role: + last_msg_content = request.messages[-1].content for choice in choices: full_message = last_msg_content + choice.message.content From ad63060708276f281bce1e34361b8b9375b09f95 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 19:59:08 +0100 Subject: [PATCH 08/16] Fix: "tools_calls" transfered on "content" when needed --- vllm/entrypoints/openai/serving.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index e727fa76fc16..c67b83351801 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -14,7 +14,7 @@ ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, DeltaMessage, LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, ChatCompletionToolParam, - ToolCallsDelta, ToolCallsMessage, FunctionCall, ErrorResponse) + ToolCallsDelta, ToolCallsMessage, FunctionCall, ChatCompletionAssistantMessage, ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer @@ -43,6 +43,12 @@ def func_call_token_size(cls) -> int: def func_call_token(cls) -> str: return "!function_call:" + def to_ChatContent(self, tool_calls: [ToolCallsMessage]) -> str: + text = "" + for call in tool_calls: + text += call.function.name + " was called with arguments : " + str(call.function.arguments) + "\n" + return text + def inject_prompt(self, request: ChatCompletionRequest): """ Tested with : https://huggingface.co/mlabonne/NeuralHermes-2.5-Mistral-7B/discussions/3 """ @@ -193,6 +199,16 @@ async def create_chat_completion( if self.openai_tools_prompter is not None: self.openai_tools_prompter.inject_prompt(request) + # FIXME : As on dec 2023, the tokenizer only accept "role" and "content" attributes. + # FIXME : So we manually copy other attributes into "content" when needed. + for m in request.messages: + if isinstance(m, ChatCompletionAssistantMessage): + if m.tool_calls is not None: + if m.content is None: + m.content = self.openai_tools_prompter.to_ChatContent(m.tool_calls) + else: + m.content += "\n" + self.openai_tools_prompter.to_ChatContent(m.tool_calls) + try: prompt = self.tokenizer.apply_chat_template( conversation=request.messages, From d1bdf419b260e1ec343e238f593535fa71494cc9 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 20 Dec 2023 20:11:32 +0100 Subject: [PATCH 09/16] format --- vllm/entrypoints/openai/serving.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index c67b83351801..c58c1c1fac61 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -14,7 +14,8 @@ ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, DeltaMessage, LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, ChatCompletionToolParam, - ToolCallsDelta, ToolCallsMessage, FunctionCall, ChatCompletionAssistantMessage, ErrorResponse) + ToolCallsDelta, ToolCallsMessage, FunctionCall, + ChatCompletionAssistantMessage, ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer @@ -46,7 +47,8 @@ def func_call_token(cls) -> str: def to_ChatContent(self, tool_calls: [ToolCallsMessage]) -> str: text = "" for call in tool_calls: - text += call.function.name + " was called with arguments : " + str(call.function.arguments) + "\n" + text += call.function.name + " was called with arguments : " + str( + call.function.arguments) + "\n" return text def inject_prompt(self, request: ChatCompletionRequest): @@ -202,12 +204,14 @@ async def create_chat_completion( # FIXME : As on dec 2023, the tokenizer only accept "role" and "content" attributes. # FIXME : So we manually copy other attributes into "content" when needed. for m in request.messages: - if isinstance(m, ChatCompletionAssistantMessage): - if m.tool_calls is not None: - if m.content is None: - m.content = self.openai_tools_prompter.to_ChatContent(m.tool_calls) - else: - m.content += "\n" + self.openai_tools_prompter.to_ChatContent(m.tool_calls) + if isinstance(m, ChatCompletionAssistantMessage + ) and m.tool_calls is not None: + if m.content is None: + m.content = self.openai_tools_prompter.to_ChatContent( + m.tool_calls) + else: + m.content += "\n" + self.openai_tools_prompter.to_ChatContent( + m.tool_calls) try: prompt = self.tokenizer.apply_chat_template( From 5a0e4eafbe439e85216c7e2e0b4a26a2ac4a657e Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Thu, 21 Dec 2023 13:12:40 +0100 Subject: [PATCH 10/16] Fix: messages roles are now literals with a discriminator --- vllm/entrypoints/openai/protocol.py | 17 +++++++++-------- vllm/entrypoints/openai/serving.py | 29 ++++++++++++++++++----------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 9d28caae193e..3ada1120bbff 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -76,36 +76,37 @@ class ChatCompletionToolParam(BaseModel): class ChatCompletionSystemMessage(BaseModel): - role: str + role: Literal["system"] content: str name: Optional[str] = None class ChatCompletionUserMessage(BaseModel): + role: Literal["user"] content: Union[str, List[str]] - role: str name: Optional[str] = None class ChatCompletionAssistantMessage(BaseModel): + role: Literal["assistant"] content: Optional[str] = None - role: str name: Optional[str] = None tool_calls: Optional[List[ToolCallsMessage]] = None class ChatCompletionToolMessage(BaseModel): + role: Literal["tool"] content: str - role: str tool_call_id: str class ChatCompletionRequest(BaseModel): model: str - messages: Union[str, List[ChatCompletionSystemMessage], - List[ChatCompletionUserMessage], - List[ChatCompletionAssistantMessage], - List[ChatCompletionToolMessage]] + messages: List[Union[ChatCompletionToolMessage, + ChatCompletionAssistantMessage, + ChatCompletionUserMessage, + ChatCompletionSystemMessage + ]] temperature: Optional[float] = 0.7 top_p: Optional[float] = 1.0 n: Optional[int] = 1 diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index c58c1c1fac61..8ad6a7aee979 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -15,7 +15,7 @@ ChatCompletionStreamResponse, ChatMessage, DeltaMessage, LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, ChatCompletionToolParam, ToolCallsDelta, ToolCallsMessage, FunctionCall, - ChatCompletionAssistantMessage, ErrorResponse) + ChatCompletionAssistantMessage, ChatCompletionToolMessage, ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.transformers_utils.tokenizer import get_tokenizer @@ -44,12 +44,19 @@ def func_call_token_size(cls) -> int: def func_call_token(cls) -> str: return "!function_call:" - def to_ChatContent(self, tool_calls: [ToolCallsMessage]) -> str: + def content_from_assistant(self, message: ChatCompletionAssistantMessage) -> str: text = "" - for call in tool_calls: - text += call.function.name + " was called with arguments : " + str( + for call in message.tool_calls: + text += call.id + " was called with arguments : " + str( call.function.arguments) + "\n" - return text + if message.content is None: + return text + else: + return message.content + "\n" + text + + + def content_from_tool(self, message: ChatCompletionToolMessage) -> str: + return message.tool_call_id + " -> " + message.content def inject_prompt(self, request: ChatCompletionRequest): """ Tested with : @@ -206,12 +213,10 @@ async def create_chat_completion( for m in request.messages: if isinstance(m, ChatCompletionAssistantMessage ) and m.tool_calls is not None: - if m.content is None: - m.content = self.openai_tools_prompter.to_ChatContent( - m.tool_calls) - else: - m.content += "\n" + self.openai_tools_prompter.to_ChatContent( - m.tool_calls) + m.content = self.openai_tools_prompter.content_from_assistant(m) + elif isinstance(m, ChatCompletionToolMessage + ) and m.tool_call_id is not None: + m.content = self.openai_tools_prompter.content_from_tool(m) try: prompt = self.tokenizer.apply_chat_template( @@ -223,6 +228,8 @@ async def create_chat_completion( f"Error in applying chat template from request: {str(e)}") return self.create_error_response(str(e)) + # logger.info(prompt) # print current prompt + token_ids, error_check_ret = await self._check_length(request, prompt=prompt) if error_check_ret is not None: From 1c3233ebaedad5a8efd7aaeb4c5e04ecc311d3b9 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Thu, 21 Dec 2023 13:27:38 +0100 Subject: [PATCH 11/16] format --- vllm/entrypoints/openai/protocol.py | 3 +-- vllm/entrypoints/openai/serving.py | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 3ada1120bbff..a2e31c830118 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -105,8 +105,7 @@ class ChatCompletionRequest(BaseModel): messages: List[Union[ChatCompletionToolMessage, ChatCompletionAssistantMessage, ChatCompletionUserMessage, - ChatCompletionSystemMessage - ]] + ChatCompletionSystemMessage]] temperature: Optional[float] = 0.7 top_p: Optional[float] = 1.0 n: Optional[int] = 1 diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index 8ad6a7aee979..c2e02ab763f5 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -44,7 +44,8 @@ def func_call_token_size(cls) -> int: def func_call_token(cls) -> str: return "!function_call:" - def content_from_assistant(self, message: ChatCompletionAssistantMessage) -> str: + def content_from_assistant(self, + message: ChatCompletionAssistantMessage) -> str: text = "" for call in message.tool_calls: text += call.id + " was called with arguments : " + str( @@ -54,7 +55,6 @@ def content_from_assistant(self, message: ChatCompletionAssistantMessage) -> str else: return message.content + "\n" + text - def content_from_tool(self, message: ChatCompletionToolMessage) -> str: return message.tool_call_id + " -> " + message.content @@ -213,7 +213,8 @@ async def create_chat_completion( for m in request.messages: if isinstance(m, ChatCompletionAssistantMessage ) and m.tool_calls is not None: - m.content = self.openai_tools_prompter.content_from_assistant(m) + m.content = self.openai_tools_prompter.content_from_assistant( + m) elif isinstance(m, ChatCompletionToolMessage ) and m.tool_call_id is not None: m.content = self.openai_tools_prompter.content_from_tool(m) From e70bc84f9cd44de8d530fe0f42a67d5bd0692c15 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 3 Jan 2024 17:38:24 +0100 Subject: [PATCH 12/16] rewrite of the output parsing, bugfixes --- vllm/entrypoints/openai/api_server.py | 1 - vllm/entrypoints/openai/serving.py | 394 +++++++++++++++----------- 2 files changed, 226 insertions(+), 169 deletions(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 0af6ad188c06..f026ad8e607f 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -143,7 +143,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request): engine_args = AsyncEngineArgs.from_cli_args(args) engine = AsyncLLMEngine.from_engine_args(engine_args) - enable_tools = True openai_tools_prompter = OpenAIToolsPrompter( ) if args.enable_api_tools else None diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index c2e02ab763f5..6dbdac9bfe9a 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -7,7 +7,7 @@ from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional, Tuple, Union from vllm.logger import init_logger from vllm.engine.async_llm_engine import AsyncLLMEngine -from vllm.entrypoints.openai.protocol import ( +from .protocol import ( CompletionRequest, CompletionResponse, CompletionResponseChoice, CompletionResponseStreamChoice, CompletionStreamResponse, ChatCompletionRequest, ChatCompletionResponse, @@ -64,29 +64,29 @@ def inject_prompt(self, request: ChatCompletionRequest): if request.tool_choice is not None and request.tools is not None and request.tool_choice == "auto": tools_list: [ChatCompletionToolParam] = request.tools if len(tools_list): - text_inject = ( - "Your task is to call a function when needed. " - "You will be provided with a list of functions. " - "\n\nAvailable functions:\n") + text_inject = "The following is a list of external functions that may be called to complete certain tasks:" + text_inject += "\n[" for tool in tools_list: if tool.type == "function": - text_inject += "\n" + tool.function.name - if tool.function.description is not None: - text_inject += " - " + tool.function.description - if tool.function.parameters is not None: - schema = json.dumps(tool.function.parameters, - indent=4) - text_inject += f"```\njsonschema\n{schema}\n```" + json_schema_params = json.dumps( + tool.function.parameters, indent=4) if ( + tool.function.parameters is not None + and len(tool.function.parameters)) else None + if json_schema_params is not None: + text_inject += f"\n {{\"name\": \"{tool.function.name}\", \"description\": \"{tool.function.description}\", \"arguments\": {json_schema_params}]}}," + else: + text_inject += f"\n {{\"name\": \"{tool.function.name}\", \"description\": \"{tool.function.description}\", \"arguments\": null]}}," + text_inject += "\n]\n" text_inject += ( - f"\nTo call a function, the response must start by" - f"\"{self.func_call_token()} followed by a json like this: " - f"{{\"call\": \"function_name\", \"params\": {{\"arg1\": \"value1\"}}}}.\n" - "If you cannot call a function due to lack of information, " - "do not make a function call and ask the user for additional details." - f"If you can call a function, you don't explain anything, just do the call. " - f"You only call functions when it's needed and when the description matches with the user input. " - f"After a function call, the response must terminate immediately.\n\n" - ) + f"Whenever the user asks you something, you can either respond directly or invoke a function. " + f"The decision to invoke a function is yours, only invoke functions when it makes sense to do so.\n" + f"If you have to call at least one function, your message can contain only function calls and nothing else.\n" + f"To call a function, the message must start by \"{self.func_call_token()}\" followed by a json like this:\n" + f"With arguments:\n" + f" {self.func_call_token()}{{\"call\": \"function_name\", \"arguments\": {{\"arg1\": \"value1\"}}}}.\n" + f"Without arguments:\n" + f" {self.func_call_token()}{{\"call\": \"function_name\", \"arguments\": null}}.\n" + f"End of functions instructions.\n\n") if isinstance(request.messages, str): request.messages = text_inject + request.messages elif isinstance(request.messages, @@ -99,13 +99,24 @@ class PromptCapture: def __init__(self): self.content: str = "" - self.ignore = False + self.maybe_function_call = False + self.is_function_call = False + self.prefix_size = 0 + self.calls_list: List[{}] = [] + + def reset(self, reset_calls_list=False): + self.content = "" + self.maybe_function_call = False self.is_function_call = False - self.calls_list: List[{}] = None + self.prefix_size = 0 + if reset_calls_list: + self.calls_list = [] + + def num_calls(self): + return len(self.calls_list) def make_calls_list(self, prompter: OpenAIToolsPrompter): calls_list = self.content.split(prompter.func_call_token()) - self.calls_list = [] for v_call in calls_list: if len(v_call): try: @@ -116,37 +127,34 @@ def make_calls_list(self, prompter: OpenAIToolsPrompter): # Simply ignore invalid functions calls... pass - def num_calls(self): - return len(self.calls_list) if self.calls_list is not None else 0 - - def calls_validation(self, tools_list: [str]) -> bool: - """ Validate function / tool calls by searching name in the tools defined in the request. """ - if self.calls_list is not None: - for ic in range(len(self.calls_list)): - if self.calls_list[ic]["call"] in tools_list: - pass - else: - return False - return True - return False + def validate_call(self, call_id: int, tools_list: [str]) -> int: + """ Validate function / tool calls by searching name in the tools defined in the request. + Returns the function id or -1 on failure.""" + if len(self.calls_list) and call_id < len(self.calls_list): + try: + return tools_list.index(self.calls_list[call_id]["call"]) + except ValueError: + pass + return -1 def to_ToolCallsMessage(self, - func_id: int) -> Union[ToolCallsMessage, None]: - if self.calls_list is not None: - call = self.calls_list[func_id] - arguments = call["params"] if "params" in call else None + call_id: int) -> Union[ToolCallsMessage, None]: + if len(self.calls_list) and call_id < len(self.calls_list): + call = self.calls_list[call_id] + arguments = call["arguments"] if "arguments" in call else None function_call = FunctionCall(name=call["call"], - arguments=json.dumps(arguments)) - return ToolCallsMessage(id="call_" + call["call"], + arguments=json.dumps(arguments) + if arguments is not None else "") + return ToolCallsMessage(id="call_" + call["call"] + "_" + + str(call_id), type="function", function=function_call) return None - def to_ToolCallsDelta(self, index: int, - func_id: int) -> Union[ToolCallsDelta, None]: - mesg = self.to_ToolCallsMessage(func_id) + def to_ToolCallsDelta(self, call_id: int) -> Union[ToolCallsDelta, None]: + mesg = self.to_ToolCallsMessage(call_id) if mesg is not None: - return ToolCallsDelta(index=index, + return ToolCallsDelta(index=call_id, id=mesg.id, type=mesg.type, function=mesg.function) @@ -164,9 +172,23 @@ def __init__(self, self.engine = engine self.openai_tools_prompter = openai_tools_prompter self.served_model = served_model + self.chat_template = chat_template self.response_role = response_role - engine_model_config = asyncio.run(engine.get_model_config()) + self.max_model_len = 0 + self.tokenizer = None + + try: + if self.engine.engine_use_ray and asyncio.get_event_loop( + ) is not None: + asyncio.create_task(self._post_init()) + else: + asyncio.run(self._post_init()) + except RuntimeError: + asyncio.run(self._post_init()) + + async def _post_init(self): + engine_model_config = await self.engine.get_model_config() self.max_model_len = engine_model_config.max_model_len # A separate tokenizer to map token IDs to strings. @@ -174,7 +196,7 @@ def __init__(self, engine_model_config.tokenizer, tokenizer_mode=engine_model_config.tokenizer_mode, trust_remote_code=engine_model_config.trust_remote_code) - self._load_chat_template(chat_template) + self._load_chat_template(self.chat_template) async def show_available_models(self) -> ModelList: """Show available models. Right now we only have one model.""" @@ -340,72 +362,53 @@ async def chat_completion_stream_generator( if finish_reason_sent[i]: continue + current_capture = tools_capture_texts[ + i] if tools_capture_texts is not None else None + # Manage tools calling if self.openai_tools_prompter is not None and \ request.tools is not None and \ - not tools_capture_texts[i].ignore: - current_capture = tools_capture_texts[i] - if (len(current_capture.content) == 0) or \ - current_capture.is_function_call or \ - current_capture.content.startswith(OpenAIToolsPrompter.func_call_token_pre()): - current_capture.content += output.text[ - len(current_capture.content):] - if len(current_capture.content) >= OpenAIToolsPrompter.func_call_token_size() and \ - not current_capture.is_function_call and \ - current_capture.content.startswith(OpenAIToolsPrompter.func_call_token()): - current_capture.is_function_call = True - if current_capture.is_function_call: - if output.finish_reason is None: - pass - elif output.finish_reason == "stop": # Send tools call - current_capture.make_calls_list( - self.openai_tools_prompter) - if not current_capture.calls_validation( - tools_list=tools_list): - current_capture.ignore = True - continue - calls_count = current_capture.num_calls() - for call_id in range(calls_count): - prompt_tokens = len(res.prompt_token_ids) - if call_id == (calls_count - 1): - final_usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=len( - output.token_ids), - total_tokens=prompt_tokens + - len(output.token_ids), - ) - else: - final_usage = None - tool_call = current_capture.to_ToolCallsDelta( - index=call_id, func_id=call_id) - choice_data = ChatCompletionResponseStreamChoice( - index=i, - delta=DeltaMessage( - content=None, - tool_calls=[tool_call]), - finish_reason="tool_calls") - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - if final_usage is not None: - chunk.usage = final_usage - data = chunk.json(exclude_unset=True, - exclude_none=True, - ensure_ascii=False) - yield f"data: {data}\n\n" - finish_reason_sent[i] = True - current_capture.ignore = True - else: - current_capture.ignore = True - else: + output.finish_reason is None: + if len(current_capture.content) == 0: + current_token: str = output.text[len(previous_texts[i] + ):] + if OpenAIToolsPrompter.func_call_token_pre( + ) in current_token: + start_pos: int = current_token.index( + OpenAIToolsPrompter.func_call_token_pre()) + current_capture.content = current_token[ + start_pos:] # With some models the completion may start by a space. + current_capture.prefix_size = len( + output.text) - len(current_capture.content) + current_capture.maybe_function_call = True + else: # Maybe a function call... + current_token: str = output.text[ + len(current_capture.content) + + current_capture.prefix_size:] + current_capture.content += current_token + if len(current_capture.content + ) < OpenAIToolsPrompter.func_call_token_size(): pass - else: - current_capture.ignore = True - else: + elif not current_capture.is_function_call: + if current_capture.content.startswith( + OpenAIToolsPrompter.func_call_token( + )): # Function call ! + current_capture.is_function_call = True + else: # This is not a function call... + current_capture.reset(False) + else: # Currently extracting the function call + if current_capture.content.rfind("}", -6) != -1: + c1 = current_capture.content.count("{") + c2 = current_capture.content.count("}") + if c1 == c2: # We have the complete call block + previous_texts[i] = output.text + current_capture.make_calls_list( + self.openai_tools_prompter) + current_capture.reset(False) + else: + pass + if current_capture is None or ( + not current_capture.maybe_function_call): if output.finish_reason is None: # Send token-by-token response for each request.n delta_text = output.text[len(previous_texts[i]):] @@ -425,30 +428,64 @@ async def chat_completion_stream_generator( ensure_ascii=False) yield f"data: {data}\n\n" else: - # Send the finish response for each request.n only once - prompt_tokens = len(res.prompt_token_ids) - final_usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=previous_num_tokens[i], - total_tokens=prompt_tokens + - previous_num_tokens[i], - ) - choice_data = ChatCompletionResponseStreamChoice( - index=i, - delta=[], - finish_reason=output.finish_reason) - chunk = ChatCompletionStreamResponse( - id=request_id, - object=chunk_object_type, - created=created_time, - choices=[choice_data], - model=model_name) - if final_usage is not None: - chunk.usage = final_usage - data = chunk.json(exclude_unset=True, - exclude_none=True, - ensure_ascii=False) - yield f"data: {data}\n\n" + if output.finish_reason == "stop" and ( + current_capture is not None and + (current_capture.num_calls() > 0)): + calls_count = current_capture.num_calls() + tools_calls_list = [] + for call_id in range(calls_count): + func_id = current_capture.validate_call( + call_id, tools_list) + if func_id >= 0: + tools_calls_list.append( + current_capture.to_ToolCallsDelta( + call_id=call_id)) + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=DeltaMessage( + content=None, tool_calls=tools_calls_list), + finish_reason="tool_calls") + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + chunk.usage = UsageInfo( + prompt_tokens=len(res.prompt_token_ids), + completion_tokens=len(output.token_ids), + total_tokens=len(res.prompt_token_ids) + + len(output.token_ids), + ) + data = chunk.json(exclude_unset=True, + exclude_none=True, + ensure_ascii=False) + yield f"data: {data}\n\n" + else: + # Send the finish response for each request.n only once + prompt_tokens = len(res.prompt_token_ids) + final_usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=previous_num_tokens[i], + total_tokens=prompt_tokens + + previous_num_tokens[i], + ) + choice_data = ChatCompletionResponseStreamChoice( + index=i, + delta=[], + finish_reason=output.finish_reason) + chunk = ChatCompletionStreamResponse( + id=request_id, + object=chunk_object_type, + created=created_time, + choices=[choice_data], + model=model_name) + if final_usage is not None: + chunk.usage = final_usage + data = chunk.json(exclude_unset=True, + exclude_none=True, + ensure_ascii=False) + yield f"data: {data}\n\n" finish_reason_sent[i] = True # Send the final done message after all response.n are finished yield "data: [DONE]\n\n" @@ -472,6 +509,10 @@ async def chat_completion_full_generator( choices = [] role = self.get_chat_request_role(request) for output in final_res.outputs: + + # logger.info(output.text) + tools_calls_validation = False + # Manage tools calling if self.openai_tools_prompter is not None and \ request.tools is not None: @@ -479,42 +520,59 @@ async def chat_completion_full_generator( tool.function.name for tool in request.tools ] current_capture = PromptCapture() - current_capture.content = output.text - if output.text.startswith( - OpenAIToolsPrompter.func_call_token_pre()): - if output.finish_reason == "stop": # Send tools call - current_capture.make_calls_list( - self.openai_tools_prompter) - if not current_capture.calls_validation( - tools_list=tools_list): - current_capture.ignore = True - else: - calls_count = current_capture.num_calls() - tool_calls = [] - for call_id in range(calls_count): - tool_calls.append( - current_capture.to_ToolCallsMessage( - func_id=call_id)) - message = ChatMessage(role=role, - content=None, - tool_calls=tool_calls) - choice_data = ChatCompletionResponseChoice( - index=output.index, - message=message, - finish_reason="tool_calls") - choices.append(choice_data) - else: - current_capture.ignore = True - else: - current_capture.ignore = True - if current_capture.ignore: + # current_capture.content = output.text.lstrip(" ") # With some models the completion may start by a space. + + start_pos = 0 + while True: + pos = output.text.find( + OpenAIToolsPrompter.func_call_token(), start_pos, -1) + if pos < 0: + break + start_bloc = output.text.find("{", pos, -1) + if start_bloc < 0: + break + if (start_bloc - + (pos + + OpenAIToolsPrompter.func_call_token_size())) > 1: + break + count = 1 + bloc_end = start_bloc + 1 + for it_ch in range(start_bloc + 1, len(output.text), 1): + ch = output.text[it_ch] + bloc_end += 1 + if ch == "{": + count += 1 + elif ch == "}": + count -= 1 + if count == 0: # We have the complete call block + current_capture.content = output.text[ + start_bloc:bloc_end] + current_capture.make_calls_list( + self.openai_tools_prompter) + current_capture.reset(False) + break + start_pos = bloc_end + 1 + + if current_capture.num_calls() > 0: + tools_calls_validation = True + calls_count = current_capture.num_calls() + tools_calls_list = [] + for call_id in range(calls_count): + func_id = current_capture.validate_call( + call_id, tools_list) + if func_id >= 0: + tools_calls_list.append( + current_capture.to_ToolCallsMessage( + call_id=call_id)) + message = ChatMessage(role=role, + content=None, + tool_calls=tools_calls_list) choice_data = ChatCompletionResponseChoice( index=output.index, - message=ChatMessage(role=role, content=output.text), - finish_reason=output.finish_reason, - ) + message=message, + finish_reason="tool_calls") choices.append(choice_data) - else: + if not tools_calls_validation: choice_data = ChatCompletionResponseChoice( index=output.index, message=ChatMessage(role=role, content=output.text), From fe4542485ca228e180504f4b6265e70394d45669 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 3 Jan 2024 18:15:01 +0100 Subject: [PATCH 13/16] API types renamed to OpenAI defined types --- vllm/entrypoints/openai/protocol.py | 20 ++++++++++---------- vllm/entrypoints/openai/serving.py | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index a2e31c830118..5c98fd12f919 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -52,18 +52,18 @@ class UsageInfo(BaseModel): completion_tokens: Optional[int] = 0 -class FunctionCall(BaseModel): +class Function(BaseModel): name: str arguments: str -class ToolCallsMessage(BaseModel): +class ChatCompletionMessageToolCall(BaseModel): id: str type: str - function: FunctionCall + function: Function -class ChatCompletionFunctionDef(BaseModel): +class FunctionDefinition(BaseModel): name: str description: str parameters: Optional[Any] = None @@ -72,7 +72,7 @@ class ChatCompletionFunctionDef(BaseModel): class ChatCompletionToolParam(BaseModel): type: str = "function" - function: ChatCompletionFunctionDef = None + function: FunctionDefinition = None class ChatCompletionSystemMessage(BaseModel): @@ -91,7 +91,7 @@ class ChatCompletionAssistantMessage(BaseModel): role: Literal["assistant"] content: Optional[str] = None name: Optional[str] = None - tool_calls: Optional[List[ToolCallsMessage]] = None + tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None class ChatCompletionToolMessage(BaseModel): @@ -203,7 +203,7 @@ class CompletionStreamResponse(BaseModel): class ChatMessage(BaseModel): role: str content: Optional[str] = None - tool_calls: Optional[List[ToolCallsMessage]] = None + tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None class ChatCompletionResponseChoice(BaseModel): @@ -221,17 +221,17 @@ class ChatCompletionResponse(BaseModel): usage: UsageInfo -class ToolCallsDelta(BaseModel): +class ChoiceDeltaToolCall(BaseModel): index: int id: str type: str - function: FunctionCall + function: Function class DeltaMessage(BaseModel): role: Optional[str] = None content: Optional[str] = None - tool_calls: Optional[List[ToolCallsDelta]] = None + tool_calls: Optional[List[ChoiceDeltaToolCall]] = None class ChatCompletionResponseStreamChoice(BaseModel): diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index 6dbdac9bfe9a..a3b2a42eeef3 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -14,7 +14,7 @@ ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, DeltaMessage, LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo, ChatCompletionToolParam, - ToolCallsDelta, ToolCallsMessage, FunctionCall, + ChoiceDeltaToolCall, ChatCompletionMessageToolCall, Function, ChatCompletionAssistantMessage, ChatCompletionToolMessage, ErrorResponse) from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams @@ -78,9 +78,9 @@ def inject_prompt(self, request: ChatCompletionRequest): text_inject += f"\n {{\"name\": \"{tool.function.name}\", \"description\": \"{tool.function.description}\", \"arguments\": null]}}," text_inject += "\n]\n" text_inject += ( - f"Whenever the user asks you something, you can either respond directly or invoke a function. " + f"Whenever the user asks you something, you can either respond directly or invoke a function.\n" f"The decision to invoke a function is yours, only invoke functions when it makes sense to do so.\n" - f"If you have to call at least one function, your message can contain only function calls and nothing else.\n" + f"If you need to call at least one function, your message should contain only a list of function calls and nothing else; the function calls are the response.\n" f"To call a function, the message must start by \"{self.func_call_token()}\" followed by a json like this:\n" f"With arguments:\n" f" {self.func_call_token()}{{\"call\": \"function_name\", \"arguments\": {{\"arg1\": \"value1\"}}}}.\n" @@ -137,24 +137,24 @@ def validate_call(self, call_id: int, tools_list: [str]) -> int: pass return -1 - def to_ToolCallsMessage(self, - call_id: int) -> Union[ToolCallsMessage, None]: + def to_ChatCompletionMessageToolCall(self, + call_id: int) -> Union[ChatCompletionMessageToolCall, None]: if len(self.calls_list) and call_id < len(self.calls_list): call = self.calls_list[call_id] arguments = call["arguments"] if "arguments" in call else None - function_call = FunctionCall(name=call["call"], + function_call = Function(name=call["call"], arguments=json.dumps(arguments) if arguments is not None else "") - return ToolCallsMessage(id="call_" + call["call"] + "_" + + return ChatCompletionMessageToolCall(id="call_" + call["call"] + "_" + str(call_id), type="function", function=function_call) return None - def to_ToolCallsDelta(self, call_id: int) -> Union[ToolCallsDelta, None]: - mesg = self.to_ToolCallsMessage(call_id) + def to_ChoiceDeltaToolCall(self, call_id: int) -> Union[ChoiceDeltaToolCall, None]: + mesg = self.to_ChatCompletionMessageToolCall(call_id) if mesg is not None: - return ToolCallsDelta(index=call_id, + return ChoiceDeltaToolCall(index=call_id, id=mesg.id, type=mesg.type, function=mesg.function) @@ -438,7 +438,7 @@ async def chat_completion_stream_generator( call_id, tools_list) if func_id >= 0: tools_calls_list.append( - current_capture.to_ToolCallsDelta( + current_capture.to_ChoiceDeltaToolCall( call_id=call_id)) choice_data = ChatCompletionResponseStreamChoice( index=i, @@ -562,7 +562,7 @@ async def chat_completion_full_generator( call_id, tools_list) if func_id >= 0: tools_calls_list.append( - current_capture.to_ToolCallsMessage( + current_capture.to_ChatCompletionMessageToolCall( call_id=call_id)) message = ChatMessage(role=role, content=None, From f0d82fab76be9aa4c8228c5026360d3b038387ee Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 3 Jan 2024 18:16:47 +0100 Subject: [PATCH 14/16] Format --- vllm/entrypoints/openai/serving.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/vllm/entrypoints/openai/serving.py b/vllm/entrypoints/openai/serving.py index a3b2a42eeef3..8c95892cd1e7 100644 --- a/vllm/entrypoints/openai/serving.py +++ b/vllm/entrypoints/openai/serving.py @@ -137,27 +137,28 @@ def validate_call(self, call_id: int, tools_list: [str]) -> int: pass return -1 - def to_ChatCompletionMessageToolCall(self, - call_id: int) -> Union[ChatCompletionMessageToolCall, None]: + def to_ChatCompletionMessageToolCall( + self, call_id: int) -> Union[ChatCompletionMessageToolCall, None]: if len(self.calls_list) and call_id < len(self.calls_list): call = self.calls_list[call_id] arguments = call["arguments"] if "arguments" in call else None function_call = Function(name=call["call"], - arguments=json.dumps(arguments) - if arguments is not None else "") - return ChatCompletionMessageToolCall(id="call_" + call["call"] + "_" + - str(call_id), - type="function", - function=function_call) + arguments=json.dumps(arguments) + if arguments is not None else "") + return ChatCompletionMessageToolCall(id="call_" + call["call"] + + "_" + str(call_id), + type="function", + function=function_call) return None - def to_ChoiceDeltaToolCall(self, call_id: int) -> Union[ChoiceDeltaToolCall, None]: + def to_ChoiceDeltaToolCall( + self, call_id: int) -> Union[ChoiceDeltaToolCall, None]: mesg = self.to_ChatCompletionMessageToolCall(call_id) if mesg is not None: return ChoiceDeltaToolCall(index=call_id, - id=mesg.id, - type=mesg.type, - function=mesg.function) + id=mesg.id, + type=mesg.type, + function=mesg.function) return None @@ -562,7 +563,8 @@ async def chat_completion_full_generator( call_id, tools_list) if func_id >= 0: tools_calls_list.append( - current_capture.to_ChatCompletionMessageToolCall( + current_capture. + to_ChatCompletionMessageToolCall( call_id=call_id)) message = ChatMessage(role=role, content=None, From f14ae0e7c1c49008de1827744eac9c345312bbbb Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 3 Jan 2024 18:52:13 +0100 Subject: [PATCH 15/16] Exemple to use tools calls --- examples/openai_tools_calls.py | 130 +++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 examples/openai_tools_calls.py diff --git a/examples/openai_tools_calls.py b/examples/openai_tools_calls.py new file mode 100755 index 000000000000..d5e6aa826894 --- /dev/null +++ b/examples/openai_tools_calls.py @@ -0,0 +1,130 @@ +""" +Inspired by the OpenAI example found here: + https://platform.openai.com/docs/guides/function-calling/parallel-function-calling +""" + +import datetime +from openai import OpenAI +import json + +client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1") +models = client.models.list() +model = models.data[0].id +stream = True + +def get_current_date_utc(): + print("Calling get_current_date_utc client side.") + return datetime.datetime.now(datetime.timezone.utc).strftime("The current UTC datetime is (day: %A, date (day/month/year): %d/%m/%Y, time: %H:%M).") + +# Example dummy function hard coded to return the same weather +# In production, this could be your backend API or an external API +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + print("Calling get_current_weather client side.") + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": unit}) + elif "san francisco" in location.lower(): + return json.dumps({"location": "San Francisco", "temperature": "72", "unit": unit}) + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": unit}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) + +def run_conversation(): + # Step 1: send the conversation and available functions to the model + # messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}] + messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris ? We also need to know the current date."}] + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_current_date_utc", + "description": "Get the current UTC time", + }, + } + ] + response = client.chat.completions.create( + model=model, + messages=messages, + tools=tools, + stream=stream, + tool_choice="auto", # auto is default, but we'll be explicit + ) + response_message = "" + tool_calls = None + if stream: + text_message = "" + for chunk in response: + if chunk.choices[0].finish_reason is not None: + if chunk.choices[0].finish_reason == "tool_calls": + tool_calls = chunk.choices[0].delta.tool_calls + break + if chunk.choices[0].delta.content is not None: + text_message += chunk.choices[0].delta.content + response_message = { "role": "assistant", "content": text_message } + else: + if not len(response.choices): + return None + response_message = response.choices[0].message + # print(str(response_message)) + tool_calls = response_message.tool_calls + + # Step 2: check if the model wanted to call a function + if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + "get_current_date_utc": get_current_date_utc, + } + messages.append(response_message) # extend conversation with assistant's reply + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + function_to_call = available_functions[function_name] + if function_name == "get_current_weather": + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + else: + function_response = function_to_call() + + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + second_response = client.chat.completions.create( + model=model, + messages=messages, + ) # get a new response from the model where it can see the function response + + for it_msg, msg in enumerate(messages): + print("Message %i:\n %s\n" % (it_msg, str(msg))) + + return second_response +result = run_conversation() +print("Final response:\n%s" % result) From c327343c1c8ca7857233c1a3a1a0f76bf79dd4f7 Mon Sep 17 00:00:00 2001 From: FlorianJoncour Date: Wed, 3 Jan 2024 18:53:40 +0100 Subject: [PATCH 16/16] Format --- examples/openai_tools_calls.py | 101 ++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/examples/openai_tools_calls.py b/examples/openai_tools_calls.py index d5e6aa826894..0ab9597d94d0 100755 --- a/examples/openai_tools_calls.py +++ b/examples/openai_tools_calls.py @@ -12,9 +12,13 @@ model = models.data[0].id stream = True + def get_current_date_utc(): print("Calling get_current_date_utc client side.") - return datetime.datetime.now(datetime.timezone.utc).strftime("The current UTC datetime is (day: %A, date (day/month/year): %d/%m/%Y, time: %H:%M).") + return datetime.datetime.now(datetime.timezone.utc).strftime( + "The current UTC datetime is (day: %A, date (day/month/year): %d/%m/%Y, time: %H:%M)." + ) + # Example dummy function hard coded to return the same weather # In production, this could be your backend API or an external API @@ -22,45 +26,65 @@ def get_current_weather(location, unit="fahrenheit"): """Get the current weather in a given location""" print("Calling get_current_weather client side.") if "tokyo" in location.lower(): - return json.dumps({"location": "Tokyo", "temperature": "10", "unit": unit}) + return json.dumps({ + "location": "Tokyo", + "temperature": "10", + "unit": unit + }) elif "san francisco" in location.lower(): - return json.dumps({"location": "San Francisco", "temperature": "72", "unit": unit}) + return json.dumps({ + "location": "San Francisco", + "temperature": "72", + "unit": unit + }) elif "paris" in location.lower(): - return json.dumps({"location": "Paris", "temperature": "22", "unit": unit}) + return json.dumps({ + "location": "Paris", + "temperature": "22", + "unit": unit + }) else: return json.dumps({"location": location, "temperature": "unknown"}) + def run_conversation(): # Step 1: send the conversation and available functions to the model # messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}] - messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris ? We also need to know the current date."}] - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + messages = [{ + "role": + "user", + "content": + "What's the weather like in San Francisco, Tokyo, and Paris ? We also need to know the current date." + }] + tools = [{ + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": + "string", + "description": + "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] }, - "required": ["location"], }, + "required": ["location"], }, }, - { - "type": "function", - "function": { - "name": "get_current_date_utc", - "description": "Get the current UTC time", - }, - } - ] + }, { + "type": "function", + "function": { + "name": "get_current_date_utc", + "description": "Get the current UTC time", + }, + }] response = client.chat.completions.create( model=model, messages=messages, @@ -79,7 +103,7 @@ def run_conversation(): break if chunk.choices[0].delta.content is not None: text_message += chunk.choices[0].delta.content - response_message = { "role": "assistant", "content": text_message } + response_message = {"role": "assistant", "content": text_message} else: if not len(response.choices): return None @@ -95,7 +119,8 @@ def run_conversation(): "get_current_weather": get_current_weather, "get_current_date_utc": get_current_date_utc, } - messages.append(response_message) # extend conversation with assistant's reply + messages.append( + response_message) # extend conversation with assistant's reply # Step 4: send the info for each function call and function response to the model for tool_call in tool_calls: function_name = tool_call.function.name @@ -109,14 +134,12 @@ def run_conversation(): else: function_response = function_to_call() - messages.append( - { - "tool_call_id": tool_call.id, - "role": "tool", - "name": function_name, - "content": function_response, - } - ) # extend conversation with function response + messages.append({ + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + }) # extend conversation with function response second_response = client.chat.completions.create( model=model, messages=messages, @@ -126,5 +149,7 @@ def run_conversation(): print("Message %i:\n %s\n" % (it_msg, str(msg))) return second_response + + result = run_conversation() print("Final response:\n%s" % result)