[LLM] update random port

Author: ltd0924

fastdeploy/engine/args_utils.py

@@ -123,9 +123,9 @@ class EngineArgs:
     Ratio of tokens to process in a block.
     """
 
-    dist_init_addr: Optional[str] = None
+    dist_init_ip: Optional[str] = None
     """
-    The master node address of multinode deployment
+    The master node ip of multinode deployment
     """
 
     nnodes: int = 1
@@ -509,7 +509,7 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
         system_group = parser.add_argument_group("System Configuration")
         system_group.add_argument(
             "--dist-init-addr",
-            default=EngineArgs.dist_init_addr,
+            default=EngineArgs.dist_init_ip,
             help=
             "IP addresses of master node.")
 
@@ -816,7 +816,7 @@ def create_engine_config(self) -> Config:
             max_num_seqs=self.max_num_seqs,
             speculative_config=speculative_cfg,
             max_num_batched_tokens=self.max_num_batched_tokens,
-            dist_init_addr=self.dist_init_addr,
+            dist_init_ip=self.dist_init_ip,
             nnodes=self.nnodes,
             node_rank=self.node_rank,
             use_warmup=self.use_warmup,

fastdeploy/engine/config.py

@@ -24,7 +24,7 @@
 from fastdeploy.platforms import current_platform
 from fastdeploy.scheduler import SchedulerConfig
 from fastdeploy.utils import (ceil_div, check_unified_ckpt, get_host_ip,
-                              is_port_available, llm_logger)
+                              is_port_available, get_random_port, llm_logger)
 
 TaskOption = Literal["generate"]
 
@@ -571,7 +571,7 @@ def __init__(
         max_model_len: int = 8192,
         max_num_seqs: int = 8,
         max_num_batched_tokens: Optional[int] = None,
-        dist_init_addr: str = None,
+        dist_init_ip: str = None,
         nnodes: int = 1,
         node_rank: int = 0,
         speculative_config: Optional[Dict[str, Any]] = None,
@@ -631,13 +631,13 @@ def __init__(
         self.max_num_batched_tokens = max_num_batched_tokens
         self.tensor_parallel_size = tensor_parallel_size
 
-        self.dist_init_addr = dist_init_addr
         self.nnode = nnodes
         self.node_rank = node_rank
-        if self.dist_init_addr is None:
+        if self.dist_init_ip is None:
             self.master_ip = "0.0.0.0"
         else:
-            self.master_ip = self.dist_init_addr.split(":")[0]
+            self.master_ip = self.dist_init_ip
+            self.dist_init_addr = f"{self.dist_init_ip}:{get_random_port()}"
 
         self.max_model_len = max_model_len
         self.max_num_seqs = max_num_seqs
@@ -713,7 +713,7 @@ def postprocess(self):
 
         self.host_ip = get_host_ip()
 
-        if self.dist_init_addr is None or self.host_ip == self.master_ip:
+        if self.dist_init_ip is None or self.host_ip == self.master_ip:
             self.is_master = True
         else:
             self.is_master = False

fastdeploy/entrypoints/openai/api_server.py

@@ -121,8 +121,8 @@ async def lifespan(app: FastAPI):
                                  args.mm_processor_kwargs, args.enable_mm,
                                  args.reasoning_parser)
     app.state.dynamic_load_weight = args.dynamic_load_weight
-    chat_handler = OpenAIServingChat(engine_client, pid, args.dist_init_addr)
-    completion_handler = OpenAIServingCompletion(engine_client, pid, args.dist_init_addr)
+    chat_handler = OpenAIServingChat(engine_client, pid, args.dist_init_ip)
+    completion_handler = OpenAIServingCompletion(engine_client, pid, args.dist_init_ip)
     engine_client.create_zmq_client(model=pid, mode=zmq.PUSH)
     engine_client.pid = pid
     app.state.engine_client = engine_client

fastdeploy/entrypoints/openai/serving_chat.py

@@ -39,16 +39,16 @@ class OpenAIServingChat:
     OpenAI-style chat completions serving
     """
 
-    def __init__(self, engine_client, pid, dist_init_addr):
+    def __init__(self, engine_client, pid, dist_init_ip):
         self.engine_client = engine_client
         self.pid = pid
-        self.master_ip = dist_init_addr
+        self.master_ip = dist_init_ip
         self.host_ip = get_host_ip()
 
     def _check_master(self):
         if self.master_ip is None:
             return True
-        if self.host_ip == self.master_ip.split(":")[0]:
+        if self.host_ip == self.master_ip
             return True
         return False
 

fastdeploy/entrypoints/openai/serving_completion.py

@@ -44,16 +44,16 @@
 
 
 class OpenAIServingCompletion:
-    def __init__(self, engine_client, pid, dist_init_addr):
+    def __init__(self, engine_client, pid, dist_init_ip):
         self.engine_client = engine_client
         self.pid = pid
-        self.master_ip = dist_init_addr
+        self.master_ip = dist_init_ip
         self.host_ip = get_host_ip()
 
     def _check_master(self):
         if self.master_ip is None:
             return True
-        if self.host_ip == self.master_ip.split(":")[0]:
+        if self.host_ip == self.master_ip
             return True
         return False
 

fastdeploy/utils.py

@@ -27,7 +27,8 @@
 from logging.handlers import BaseRotatingHandler
 from pathlib import Path
 from typing import Literal, TypeVar, Union
-
+import random
+import socket
 import requests
 import yaml
 from aistudio_sdk.snapshot_download import snapshot_download
@@ -443,6 +444,19 @@ def get_host_ip():
     return ip
 
 
+
+
+def get_random_port():
+    while True:
+        port = random.randint(49152, 65535)
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.bind(("0.0.0.0", port)) 
+                return port 
+            except OSError:
+                continue
+
+
 def is_port_available(host, port):
     """
     Check the port is available