From b04f8c6ce74317ed0cf5e0a992b87cf21d851f13 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:55:38 +0800 Subject: [PATCH 1/4] minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: MiniMax-M3.1-Flash TP8/TEP8/DEP8 on two GB300 trays --- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 139 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 51 +++++++ inferencex-e2e/perf-changelog.yaml | 8 + 3 files changed, 198 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..dbbe545a83 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,139 @@ +schema: 2 + +# MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs), aggregated through +# Dynamo vLLM. Serve flags follow the single-node GB300 recipe +# (single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp); the +# Dynamo setup follows minimaxm3/vllm/gb200-fp4/agentx. +base: + identity: + model: {} + container: {} + frameworks: {dynamo: "1.5.0.dev20260908"} + dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} + environment: {ETCD_LEASE_TTL: "7200"} + slurm: {time_limit: "12:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + # Rendering the chat template for long AgentX conversations exceeds the + # 30 s default under concurrent load; vLLM then aborts the request. + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: "600" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "@fabric.rdma-device-ports" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "@fabric.rdma-devices" + args: + served-model-name: "MiniMaxAI/MiniMax-M3.1-Flash" + pipeline-parallel-size: 1 + trust-remote-code: true + language-model-only: true + enable-prefix-caching: true + kv-cache-metrics: true + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: "nvfp4" + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-model-len: 1048576 + max-num-batched-tokens: 8192 + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# TP8 across both trays; seven-token DSpark (golden AL 4.04). +override_tp8: + name: "minimaxm31flash-vllm-agg-gb300-tp8-agentic" + roles: + agg: + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 + max-num-seqs: 64 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +# TEP8: TP8 attention with expert-parallel MoE; seven-token DSpark (golden AL 4.04). +override_tep8: + name: "minimaxm31flash-vllm-agg-gb300-tep8-agentic" + roles: + agg: + env: + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + max-num-seqs: 128 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +# DEP8: TP1 x DP8 + EP8, one DP launch per tray, with vLLM Simple CPU KV +# offload. Each DP rank gets a quarter of its tray's DRAM allowance (179 GiB of +# the 771 GB that dram-utilization 0.80 leaves per tray); three-token DSpark +# (golden AL 2.88). +override_dep8_vllm_simple: + name: "minimaxm31flash-vllm-agg-gb300-dep8-vllm-simple-agentic" + engine: + dp_launch_mode: per_node + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + max-num-seqs: 32 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":192199557120,"cpu_bytes_to_use_per_rank":192199557120,"lazy_offload":true}}' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 804ca5a0b1..3662226c51 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5608,6 +5608,57 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } +# MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) through Dynamo vLLM: +# TP8 for low concurrency, TEP8 for the middle, and DEP8 with vLLM Simple CPU KV +# offload at the top. DSpark on the FlashInfer PrimiTS drafter as in the +# single-node GB300 recipe (7 tokens TP8/TEP8, 3 tokens DEP8). +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.5.0.dev20260908" } + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 4, 8, 16, 32] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: none + conc-list: [48, 64] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tep8 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [72, 96, 128] + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_dep8_vllm_simple + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 120531df64..1ef93e1124 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10988,3 +10988,11 @@ - "Add day-0 MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4 experts + NVFP4 KV cache) AgentX on GB200, matching the B200 recipe (#4426) with the arm64 build of the same day-0 vLLM image (minimax-m31-flash-arm64-cu134)." - "DSpark with the built-in drafter (probabilistic drafting, FLASH_ATTN drafter, fp8 drafter KV); TP4 c1-c32 drafting 7 tokens (golden AL 4.04); DEP4 (TP1 x DP4 + EP4 behind vllm-router) c64-c128 drafting 3 tokens (golden AL 2.88)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4433 + +- config-keys: + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg + scenario-type: [agentic-coding] + description: + - "Add MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) aggregated through Dynamo vLLM, with the single-node GB300 serve flags (NVFP4 KV and indexer cache, 600 s chat-template render timeout) and the day-0 arm64 vLLM image." + - "TP8 c1-c32 and TEP8 (TP8 + EP8) c48-c64 with 7-token DSpark (golden AL 4.04); DEP8 (TP1 x DP8 + EP8, per-node DP launch) c72-c128 with 3-token DSpark (golden AL 2.88) and vLLM Simple CPU KV offload (179 GiB per DP rank, lazy). FlashInfer PrimiTS drafter throughout." + pr-link: TBD From 9923a77b7c676e23cb48cfdac5510dbc15fe9ff9 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:55:48 +0800 Subject: [PATCH 2/4] perf-changelog: fill minimaxm31flash gb300 dynamo pr-link --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1ef93e1124..5eaed6252b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10995,4 +10995,4 @@ description: - "Add MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) aggregated through Dynamo vLLM, with the single-node GB300 serve flags (NVFP4 KV and indexer cache, 600 s chat-template render timeout) and the day-0 arm64 vLLM image." - "TP8 c1-c32 and TEP8 (TP8 + EP8) c48-c64 with 7-token DSpark (golden AL 4.04); DEP8 (TP1 x DP8 + EP8, per-node DP launch) c72-c128 with 3-token DSpark (golden AL 2.88) and vLLM Simple CPU KV offload (179 GiB per DP rank, lazy). FlashInfer PrimiTS drafter throughout." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4436 From a89c03ab3d4ed11d3d1c0a824372c70b523f3828 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:11:10 +0800 Subject: [PATCH 3/4] minimaxm31flash gb300 dynamo agg: Dynamo 1.6.0.dev20261016 for the vLLM engine-helper rename --- .../minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml | 6 ++++-- inferencex-e2e/configs/nvidia-master.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml index dbbe545a83..9b92f71141 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -8,8 +8,10 @@ base: identity: model: {} container: {} - frameworks: {dynamo: "1.5.0.dev20260908"} - dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} + frameworks: {dynamo: "1.6.0.dev20261016"} + # 1.6.0.dev20261014+ imports the vLLM engine-launch helpers lazily + # (ai-dynamo/dynamo#16378); earlier builds fail at import on this vLLM. + dynamo: {install: true, source: {pypi: "1.6.0.dev20261016"}} environment: {ETCD_LEASE_TTL: "7200"} slurm: {time_limit: "12:00:00"} health_check: {max_attempts: 2160, interval_seconds: 10} diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 3662226c51..443df8ec8b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5619,7 +5619,7 @@ minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.5.0.dev20260908" } + router: { name: dynamo-router, version: "1.6.0.dev20261016" } multinode: true disagg: false srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx From ce61e2453b4c6f78a2bb77b52e5d1c577ea7ac89 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:33:53 +0800 Subject: [PATCH 4/4] minimaxm31flash gb300 dynamo agg: DEP8 only (NVFP4 KV attention cannot run at TP8) --- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 29 ++----------------- inferencex-e2e/configs/nvidia-master.yaml | 27 +++-------------- inferencex-e2e/perf-changelog.yaml | 3 +- 3 files changed, 9 insertions(+), 50 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml index 9b92f71141..486fee3239 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -1,7 +1,9 @@ schema: 2 # MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs), aggregated through -# Dynamo vLLM. Serve flags follow the single-node GB300 recipe +# Dynamo vLLM. DEP8 only: the NVFP4 KV-cache attention kernels need 16 query +# heads per KV head on each rank (64 query / 4 KV heads), so attention TP8 does +# not start; low and mid concurrency stay on the single-tray TP4 recipe. Serve flags follow the single-node GB300 recipe # (single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp); the # Dynamo setup follows minimaxm3/vllm/gb200-fp4/agentx. base: @@ -95,31 +97,6 @@ base: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" -# TP8 across both trays; seven-token DSpark (golden AL 4.04). -override_tp8: - name: "minimaxm31flash-vllm-agg-gb300-tp8-agentic" - roles: - agg: - env: - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - args: - tensor-parallel-size: 8 - max-num-seqs: 64 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - -# TEP8: TP8 attention with expert-parallel MoE; seven-token DSpark (golden AL 4.04). -override_tep8: - name: "minimaxm31flash-vllm-agg-gb300-tep8-agentic" - roles: - agg: - env: - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - args: - tensor-parallel-size: 8 - enable-expert-parallel: true - max-num-seqs: 128 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - # DEP8: TP1 x DP8 + EP8, one DP launch per tray, with vLLM Simple CPU KV # offload. Each DP rank gets a quarter of its tray's DRAM allowance (179 GiB of # the 771 GB that dram-utilization 0.80 leaves per tray); three-token DSpark diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2c0cc35350..d46fe0a069 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5609,9 +5609,10 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) through Dynamo vLLM: -# TP8 for low concurrency, TEP8 for the middle, and DEP8 with vLLM Simple CPU KV -# offload at the top. DSpark on the FlashInfer PrimiTS drafter as in the -# single-node GB300 recipe (7 tokens TP8/TEP8, 3 tokens DEP8). +# DEP8 (TP1 x DP8 + EP8) with vLLM Simple CPU KV offload for high concurrency, +# 3-token DSpark on the FlashInfer PrimiTS drafter. Attention TP8 is not +# possible with the NVFP4 KV cache (16 query heads per KV head per rank needed), +# so low and mid concurrency stay on minimaxm31flash-fp4-gb300-vllm-agentic-dspark. minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 model: MiniMaxAI/MiniMax-M3.1-Flash @@ -5627,26 +5628,6 @@ minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - kv-offloading: none - conc-list: [1, 4, 8, 16, 32] - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8 - worker: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: none - conc-list: [48, 64] - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tep8 - worker: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 4c94976c31..d71eb6c21a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11003,5 +11003,6 @@ scenario-type: [agentic-coding] description: - "Add MiniMax-M3.1-Flash AgentX on two GB300 trays (8 GPUs) aggregated through Dynamo vLLM, with the single-node GB300 serve flags (NVFP4 KV and indexer cache, 600 s chat-template render timeout) and the day-0 arm64 vLLM image." - - "TP8 c1-c32 and TEP8 (TP8 + EP8) c48-c64 with 7-token DSpark (golden AL 4.04); DEP8 (TP1 x DP8 + EP8, per-node DP launch) c72-c128 with 3-token DSpark (golden AL 2.88) and vLLM Simple CPU KV offload (179 GiB per DP rank, lazy). FlashInfer PrimiTS drafter throughout." + - "DEP8 (TP1 x DP8 + EP8, per-node DP launch) c72/c96/c128 with 3-token DSpark on the FlashInfer PrimiTS drafter (golden AL 2.88) and vLLM Simple CPU KV offload (179 GiB per DP rank, lazy), on Dynamo 1.6.0.dev20261016." + - "No TP8/TEP8 points: vLLM's NVFP4 KV-cache attention needs 16 query heads per KV head on each rank (64 query / 4 KV heads), which attention TP8 halves to 8." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4436