Address brixbench review feedback

Signed-off-by: Misun Park <misuneeh@gmail.com>

Author: whalepark

brixbench/benchmark/runner_fallback_test.go

@@ -17,7 +17,9 @@ limitations under the License.
 package benchmark
 
 import (
+	"context"
 	"errors"
+	"strings"
 	"testing"
 
 	"github.com/vllm-project/aibrix/brixbench/internal/resolver"
@@ -142,3 +144,25 @@ func TestShouldRunDynamoStaleCleanupRequiresResetEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestSetupAndRunDeploymentRejectsLLMdProvider(t *testing.T) {
+	provider := "llmd"
+	testCase := &resolver.Test{
+		Name:     "manual-llmd",
+		Provider: &provider,
+	}
+
+	deployer, gatewayURL, err := setupAndRunDeployment(context.Background(), t, t.TempDir(), testCase, "benchmark", t.TempDir())
+	if err == nil {
+		t.Fatalf("expected llmd not implemented error")
+	}
+	if deployer != nil {
+		t.Fatalf("expected nil deployer, got %T", deployer)
+	}
+	if gatewayURL != "" {
+		t.Fatalf("expected empty gateway URL, got %q", gatewayURL)
+	}
+	if !strings.Contains(err.Error(), "provider llmd is not implemented") {
+		t.Fatalf("expected llmd not implemented error, got %v", err)
+	}
+}

brixbench/benchmark/runner_test.go

@@ -200,8 +200,7 @@ func setupAndRunDeployment(ctx context.Context, t *testing.T, projectRoot string
 		deployer = deployers.NewAIBrixDeployer()
 		t.Log("Using AIBrix deployer")
 	case "llmd":
-		t.Log("Using LLM-d deployer")
-		// return nil, "", fmt.Errorf("LLM-d deployer not implemented")
+		return nil, "", fmt.Errorf("provider llmd is not implemented")
 	case "dynamo":
 		deployer = deployers.NewDynamoDeployer()
 		t.Log("Using Dynamo deployer")

brixbench/benchmark/testdata/deployments/dynamo/qwen3-32b-round-robin-4p8d-vke.yaml

@@ -195,8 +195,7 @@ spec:
                 --gpu-memory-utilization $GPU_MEM_UTIL \
                 --tensor-parallel-size $TP \
                 --disaggregation-mode decode \
-                --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
-                --enforce-eager
+                --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
           env:
             - name: DYN_LOG
               value: "info"
@@ -298,8 +297,7 @@ spec:
                 --tensor-parallel-size $TP \
                 --disaggregation-mode prefill \
                 --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
-                --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \
-                --enforce-eager
+                --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
           env:
             - name: DYN_LOG
               value: "info"

brixbench/benchmark/testdata/deployments/dynamo/qwen3-8b-round-robin-1p1d-tp2-vke.yaml

@@ -169,8 +169,7 @@ spec:
                 --gpu-memory-utilization $GPU_MEM_UTIL \
                 --tensor-parallel-size $TP \
                 --disaggregation-mode decode \
-                --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
-                --enforce-eager
+                --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
           env:
             - name: DYN_LOG
               value: "info"
@@ -255,8 +254,7 @@ spec:
                 --tensor-parallel-size $TP \
                 --disaggregation-mode prefill \
                 --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \
-                --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \
-                --enforce-eager
+                --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
           env:
             - name: DYN_LOG
               value: "info"

brixbench/internal/resolver/provider_inputs.go

@@ -21,6 +21,9 @@ func validateProviderInputs(test *Test) error {
 }
 
 func validateNullProviderSourceSelection(test *Test) error {
+	if strings.TrimSpace(test.Version) != "" || strings.TrimSpace(test.Commit) != "" || len(test.ControlPlane) > 0 || strings.TrimSpace(test.Platform.ValuesFile) != "" {
+		return fmt.Errorf("provider null does not support version, commit, controlplane, or platform inputs for %s", test.Name)
+	}
 	if strings.TrimSpace(test.LocalPath) != "" {
 		return fmt.Errorf("localPath is only supported for provider aibrix in %s", test.Name)
 	}

brixbench/internal/resolver/provider_inputs_test.go

@@ -13,6 +13,40 @@ func TestValidateProviderInputsAcceptsNullProvider(t *testing.T) {
 	}
 }
 
+func TestValidateProviderInputsRejectsNullProviderUnsupportedInputs(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		test Test
+	}{
+		{
+			name: "version",
+			test: Test{Name: "baseline", Version: "v0.6.0"},
+		},
+		{
+			name: "commit",
+			test: Test{Name: "baseline", Commit: "abcdef0"},
+		},
+		{
+			name: "controlplane",
+			test: Test{Name: "baseline", ControlPlane: []string{"controlplane.yaml"}},
+		},
+		{
+			name: "platform values",
+			test: Test{Name: "baseline", Platform: Platform{ValuesFile: "platform.yaml"}},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateProviderInputs(&tc.test)
+			if err == nil {
+				t.Fatalf("expected provider null unsupported input error")
+			}
+			if !strings.Contains(err.Error(), "provider null does not support version, commit, controlplane, or platform inputs") {
+				t.Fatalf("expected provider null unsupported input error, got %v", err)
+			}
+		})
+	}
+}
+
 func TestValidateProviderInputsRejectsLLMdProvider(t *testing.T) {
 	provider := "llmd"
 	test := Test{Name: "llmd", Provider: &provider}