[Bug Fix] disable prefix caching in mm model (#4167)

* add http get retry

* fix coments

* disable prefix caching in mm model

* fix unit test

---------

Co-authored-by: zhangjunjun04 <zhangjunjun04@baidu.com>
Co-authored-by: Jiang-Jia-Jun <163579578+Jiang-Jia-Jun@users.noreply.github.com>
Co-authored-by: YuBaoku <49938469+EmmonsCurse@users.noreply.github.com>

Author: 4 people

fastdeploy/config.py

@@ -1540,6 +1540,8 @@ def postprocess(self):
 
         self.cache_config.postprocess(self.scheduler_config.max_num_batched_tokens, self.scheduler_config.max_num_seqs)
         self.cache_config.max_block_num_per_seq = int(self.max_model_len // self.cache_config.block_size)
+        if self.model_config is not None and self.model_config.enable_mm:
+            self.cache_config.enable_prefix_caching = False
 
         if self.guided_decoding_backend == "auto":
             if current_platform.is_xpu() or self.speculative_config.method is not None:

fastdeploy/engine/args_utils.py

@@ -409,8 +409,6 @@ def __post_init__(self):
             self.enable_prefix_caching = False
         if self.speculative_config is not None:
             self.enable_prefix_caching = False
-        if self.enable_mm:
-            self.enable_prefix_caching = False
         if not current_platform.is_cuda():
             self.enable_prefix_caching = False
         if self.dynamic_load_weight: