Skip to content

Commit 0248172

Browse files
shmuelkshmuelk
authored
Have a default configuration for the EndpointPicker (#2341)
* Added a function to create the default EPP configuration Signed-off-by: Shmuel Kallner <kallner@il.ibm.com> * If no configuration is provided use the default EPP configuration Signed-off-by: Shmuel Kallner <kallner@il.ibm.com> * Remove command line argument checks that force a configuration Signed-off-by: Shmuel Kallner <kallner@il.ibm.com> * Added a test of using the default EPP configuration Signed-off-by: Shmuel Kallner <kallner@il.ibm.com> --------- Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>
1 parent 2b4bcf5 commit 0248172

5 files changed

Lines changed: 109 additions & 11 deletions

File tree

‎cmd/epp/runner/runner.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -444,10 +444,6 @@ func (r *Runner) registerInTreePlugins() {
444444
}
445445

446446
func (r *Runner) parseConfigurationPhaseOne(ctx context.Context, opts *runserver.Options) (*configapi.EndpointPickerConfig, error) {
447-
if opts.ConfigText == "" && opts.ConfigFile == "" {
448-
return nil, nil // configuring through code, not through file
449-
}
450-
451447
logger := log.FromContext(ctx)
452448

453449
var configBytes []byte

‎pkg/epp/config/loader/configloader.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,19 @@ func RegisterFeatureGate(gate string) {
5555
// LoadRawConfig parses the raw configuration bytes, applies initial defaults, and extracts feature gates.
5656
// It does not instantiate plugins.
5757
func LoadRawConfig(configBytes []byte, logger logr.Logger) (*configapi.EndpointPickerConfig, map[string]bool, error) {
58-
rawConfig, err := decodeRawConfig(configBytes)
59-
if err != nil {
60-
return nil, nil, err
58+
var rawConfig *configapi.EndpointPickerConfig
59+
var err error
60+
if len(configBytes) != 0 {
61+
rawConfig, err = decodeRawConfig(configBytes)
62+
if err != nil {
63+
return nil, nil, err
64+
}
65+
logger.Info("Loaded raw configuration", "config", rawConfig.String())
66+
} else {
67+
logger.Info("A configuration wasn't specified. A default one is being used.")
68+
rawConfig = loadDefaultConfig()
69+
logger.Info("Default raw configuration used", "config", rawConfig.String())
6170
}
62-
logger.Info("Loaded raw configuration", "config", rawConfig)
6371

6472
applyStaticDefaults(rawConfig)
6573

‎pkg/epp/config/loader/configloader_test.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ import (
4343
framework "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/interface/scheduling"
4444
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/picker"
4545
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/profile"
46+
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/scorer"
47+
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/scorer/prefix"
4648
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector/framework/plugins/utilizationdetector"
4749
"sigs.k8s.io/gateway-api-inference-extension/test/utils"
4850
)
@@ -67,6 +69,10 @@ func TestLoadRawConfiguration(t *testing.T) {
6769
RegisterFeatureGate(datalayer.ExperimentalDatalayerFeatureGate)
6870
RegisterFeatureGate(flowcontrol.FeatureGate)
6971

72+
queueScorerWeight := 2.0
73+
kvCacheUtilizationScorerWeight := 2.0
74+
prefixCacheScorerWeight := 3.0
75+
7076
tests := []struct {
7177
name string
7278
configText string
@@ -124,6 +130,51 @@ func TestLoadRawConfiguration(t *testing.T) {
124130
},
125131
wantErr: false,
126132
},
133+
{
134+
name: "Success - Default configuration",
135+
configText: "",
136+
want: &configapi.EndpointPickerConfig{
137+
TypeMeta: metav1.TypeMeta{
138+
APIVersion: "inference.networking.x-k8s.io/v1alpha1",
139+
Kind: "EndpointPickerConfig",
140+
},
141+
FeatureGates: configapi.FeatureGates{},
142+
Plugins: []configapi.PluginSpec{
143+
{
144+
Name: scorer.QueueScorerType,
145+
Type: scorer.QueueScorerType,
146+
},
147+
{
148+
Name: scorer.KvCacheUtilizationScorerType,
149+
Type: scorer.KvCacheUtilizationScorerType,
150+
},
151+
{
152+
Name: prefix.PrefixCachePluginType,
153+
Type: prefix.PrefixCachePluginType,
154+
},
155+
},
156+
SchedulingProfiles: []configapi.SchedulingProfile{
157+
{
158+
Name: "default",
159+
Plugins: []configapi.SchedulingPlugin{
160+
{
161+
PluginRef: scorer.QueueScorerType,
162+
Weight: &queueScorerWeight,
163+
},
164+
{
165+
PluginRef: scorer.KvCacheUtilizationScorerType,
166+
Weight: &kvCacheUtilizationScorerWeight,
167+
},
168+
{
169+
PluginRef: prefix.PrefixCachePluginType,
170+
Weight: &prefixCacheScorerWeight,
171+
},
172+
},
173+
},
174+
},
175+
},
176+
wantErr: false,
177+
},
127178
{
128179
name: "Error - Invalid YAML",
129180
configText: errorBadYamlText,

‎pkg/epp/config/loader/defaults.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,65 @@ package loader
1919
import (
2020
"fmt"
2121

22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
2224
configapi "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1"
2325
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/flowcontrol/registry"
2426
fwkplugin "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/interface/plugin"
2527
framework "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/interface/scheduling"
2628
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/picker"
2729
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/profile"
30+
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/scorer"
31+
"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/framework/plugins/scheduling/scorer/prefix"
2832
)
2933

3034
// DefaultScorerWeight is the weight used for scorers referenced in the configuration without explicit weights.
3135
const DefaultScorerWeight = 1.0
3236

3337
var defaultScorerWeight = DefaultScorerWeight
3438

39+
func loadDefaultConfig() *configapi.EndpointPickerConfig {
40+
queueScorerWeight := 2.0
41+
kvCacheUtilizationScorerWeight := 2.0
42+
prefixCacheScorerWeight := 3.0
43+
return &configapi.EndpointPickerConfig{
44+
TypeMeta: metav1.TypeMeta{
45+
APIVersion: "inference.networking.x-k8s.io/v1alpha1",
46+
Kind: "EndpointPickerConfig",
47+
},
48+
Plugins: []configapi.PluginSpec{
49+
{
50+
Type: scorer.QueueScorerType,
51+
},
52+
{
53+
Type: scorer.KvCacheUtilizationScorerType,
54+
},
55+
{
56+
Type: prefix.PrefixCachePluginType,
57+
},
58+
},
59+
SchedulingProfiles: []configapi.SchedulingProfile{
60+
{
61+
Name: "default",
62+
Plugins: []configapi.SchedulingPlugin{
63+
{
64+
PluginRef: scorer.QueueScorerType,
65+
Weight: &queueScorerWeight,
66+
},
67+
{
68+
PluginRef: scorer.KvCacheUtilizationScorerType,
69+
Weight: &kvCacheUtilizationScorerWeight,
70+
},
71+
{
72+
PluginRef: prefix.PrefixCachePluginType,
73+
Weight: &prefixCacheScorerWeight,
74+
},
75+
},
76+
},
77+
},
78+
}
79+
}
80+
3581
// applyStaticDefaults sanitizes the configuration object before plugin instantiation.
3682
// It handles "Static" defaults: simple structural changes to the API object that do not require access to the plugin
3783
// registry.

‎pkg/epp/server/options.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -228,9 +228,6 @@ func (opts *Options) Validate() error {
228228
}
229229
}
230230

231-
if opts.ConfigText == "" && opts.ConfigFile == "" {
232-
return fmt.Errorf("one of the %q and %q flags must be set", "configText", "configFile")
233-
}
234231
if opts.ConfigText != "" && opts.ConfigFile != "" {
235232
return fmt.Errorf("both the %q and %q flags can not be set at the same time", "configText", "configFile")
236233
}

0 commit comments

Comments
 (0)