From 65f2affc5ad2eb25d715ee3a96697fc3abdd888e Mon Sep 17 00:00:00 2001 From: Ignasi Barrera Date: Mon, 22 Jun 2026 22:07:31 +0200 Subject: [PATCH] controller: add consistency check for gateway object namespaces Signed-off-by: Ignasi Barrera --- cmd/aigw/translate.go | 2 +- cmd/controller/main.go | 8 ++ cmd/controller/main_test.go | 3 + internal/controller/controller.go | 5 +- internal/controller/gateway.go | 83 +++++++++++-------- internal/controller/gateway_test.go | 59 +++++++++---- .../ai-gateway-helm/templates/deployment.yaml | 1 + 7 files changed, 108 insertions(+), 53 deletions(-) diff --git a/cmd/aigw/translate.go b/cmd/aigw/translate.go index 12f11ecbf6..eb0d20876f 100644 --- a/cmd/aigw/translate.go +++ b/cmd/aigw/translate.go @@ -247,7 +247,7 @@ func translateCustomResourceObjects( mcpC := controller.NewMCPRouteController(fakeClient, fakeClientSet, logr.FromSlogHandler(logger.Handler()), make(chan event.GenericEvent, eventChanBuffer), ) - gwC := controller.NewGatewayController(fakeClient, fakeClientSet, logr.FromSlogHandler(logger.Handler()), + gwC := controller.NewGatewayController(fakeClient, fakeClientSet, logr.FromSlogHandler(logger.Handler()), "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "debug", true, func() string { return "aigw-translate" }, false, diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 627b21c28f..0cf92704b6 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -38,6 +38,7 @@ import ( ) type flags struct { + envoyGatewayNamespace string extProcLogLevel string extProcEnableRedaction bool extProcImage string @@ -108,6 +109,11 @@ func parseWatchNamespaces(s string) []string { func parseAndValidateFlags(args []string) (*flags, error) { fs := flag.NewFlagSet("AI Gateway Controller", flag.ContinueOnError) + envoyGatewayNamespace := fs.String( + "envoyGatewayNamespace", + "envoy-gateway-system", + "The namespace where Envoy Gateway is deployed.", + ) extProcLogLevelPtr := fs.String( "extProcLogLevel", "info", @@ -328,6 +334,7 @@ func parseAndValidateFlags(args []string) (*flags, error) { } return &flags{ + envoyGatewayNamespace: *envoyGatewayNamespace, extProcLogLevel: *extProcLogLevelPtr, extProcEnableRedaction: *extProcEnableRedactionPtr, extProcImage: *extProcImagePtr, @@ -444,6 +451,7 @@ func main() { // Start the controller. if err := controller.StartControllers(ctx, mgr, k8sConfig, ctrl.Log.WithName("controller"), &controller.Options{ + EnvoyGatewayNamespace: parsedFlags.envoyGatewayNamespace, ExtProcImage: parsedFlags.extProcImage, ExtProcImagePullPolicy: parsedFlags.extProcImagePullPolicy, ExtProcLogLevel: parsedFlags.extProcLogLevel, diff --git a/cmd/controller/main_test.go b/cmd/controller/main_test.go index 9ab36f72c5..a48bc080fe 100644 --- a/cmd/controller/main_test.go +++ b/cmd/controller/main_test.go @@ -22,6 +22,7 @@ import ( func Test_parseAndValidateFlags(t *testing.T) { t.Run("no flags", func(t *testing.T) { f, err := parseAndValidateFlags([]string{}) + require.Equal(t, "envoy-gateway-system", f.envoyGatewayNamespace) require.Equal(t, "info", f.extProcLogLevel) require.False(t, f.extProcEnableRedaction) require.Equal(t, "docker.io/envoyproxy/ai-gateway-extproc:latest", f.extProcImage) @@ -48,6 +49,7 @@ func Test_parseAndValidateFlags(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { args := []string{ + tc.dash + "envoyGatewayNamespace=eg-system", tc.dash + "extProcLogLevel=debug", tc.dash + "extProcEnableRedaction=true", tc.dash + "extProcImage=example.com/extproc:latest", @@ -70,6 +72,7 @@ func Test_parseAndValidateFlags(t *testing.T) { tc.dash + "mcpFallbackSessionEncryptionIterations=200", } f, err := parseAndValidateFlags(args) + require.Equal(t, "eg-system", f.envoyGatewayNamespace) require.Equal(t, "debug", f.extProcLogLevel) require.True(t, f.extProcEnableRedaction) require.Equal(t, "example.com/extproc:latest", f.extProcImage) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 9af761dbc7..3c98c72b30 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -106,6 +106,8 @@ type Options struct { EndpointPrefixes string // RateLimitRunner is the xDS runner that serves rate limit configs to the rate limit service. RateLimitRunner *runner.Runner + // EnvoyGatewayNamespace is the namespace where Envoy Gateway is deployed. + EnvoyGatewayNamespace string } // StartControllers starts the controllers for the AI Gateway. @@ -127,7 +129,8 @@ func StartControllers(ctx context.Context, mgr manager.Manager, config *rest.Con gatewayEventChan := make(chan event.GenericEvent, 100) gatewayC := NewGatewayController(c, kubernetes.NewForConfigOrDie(config), - logger.WithName("gateway"), options.ExtProcImage, options.ExtProcLogLevel, false, uuid.NewString, isKubernetes133OrLater(versionInfo, logger)) + logger.WithName("gateway"), options.EnvoyGatewayNamespace, options.ExtProcImage, options.ExtProcLogLevel, + false, uuid.NewString, isKubernetes133OrLater(versionInfo, logger)) if err = TypedControllerBuilderForCRD(mgr, &gwapiv1.Gateway{}). WatchesRawSource(source.Channel( gatewayEventChan, diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index 12acd9ae38..6e331f0f8e 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -49,7 +49,7 @@ const ( // extProcImage is the image of the external processor sidecar container which will be used // to check if the pods of the gateway deployment need to be rolled out. func NewGatewayController( - client client.Client, kube kubernetes.Interface, logger logr.Logger, + client client.Client, kube kubernetes.Interface, logger logr.Logger, envoyGatewayNamespace string, extProcImage string, extProcLogLevel string, standAlone bool, uuidFn func() string, extProcAsSideCar bool, ) *GatewayController { uf := uuidFn @@ -57,24 +57,26 @@ func NewGatewayController( uf = uuid.NewString } return &GatewayController{ - client: client, - kube: kube, - logger: logger, - extProcImage: extProcImage, - extProcLogLevel: extProcLogLevel, - standAlone: standAlone, - uuidFn: uf, - extProcAsSideCar: extProcAsSideCar, + client: client, + kube: kube, + logger: logger, + envoyGatewayNamespace: envoyGatewayNamespace, + extProcImage: extProcImage, + extProcLogLevel: extProcLogLevel, + standAlone: standAlone, + uuidFn: uf, + extProcAsSideCar: extProcAsSideCar, } } // GatewayController implements reconcile.TypedReconciler for gwapiv1.Gateway. type GatewayController struct { - client client.Client - kube kubernetes.Interface - logger logr.Logger - extProcImage string // The image of the external processor sidecar container. - extProcLogLevel string // The log level for the extproc container. + client client.Client + kube kubernetes.Interface + logger logr.Logger + envoyGatewayNamespace string // The namespace where Envoy Gateway is deployed. + extProcImage string // The image of the external processor sidecar container. + extProcLogLevel string // The log level for the extproc container. // standAlone indicates whether the controller is running in standalone mode. standAlone bool uuidFn func() string // Function to generate a new UUID for the filter config. @@ -1145,32 +1147,45 @@ func (c *GatewayController) getObjectsForGateway(ctx context.Context, gw *gwapiv listOption := metav1.ListOptions{LabelSelector: fmt.Sprintf( "%s=%s,%s=%s", egOwningGatewayNameLabel, gw.Name, egOwningGatewayNamespaceLabel, gw.Namespace, )} - var ps *corev1.PodList - ps, err = c.kube.CoreV1().Pods("").List(ctx, listOption) - if err != nil { - err = fmt.Errorf("failed to list pods: %w", err) - return - } - pods = ps.Items - var ds *appsv1.DeploymentList - ds, err = c.kube.AppsV1().Deployments("").List(ctx, listOption) - if err != nil { - err = fmt.Errorf("failed to list deployments: %w", err) - return + var distinctNamespaces []string + for _, ns := range []string{gw.Namespace, c.envoyGatewayNamespace} { + var ps *corev1.PodList + ps, err = c.kube.CoreV1().Pods(ns).List(ctx, listOption) + if err != nil { + err = fmt.Errorf("failed to list pods in namespace %s: %w", ns, err) + return + } + pods = append(pods, ps.Items...) + + var ds *appsv1.DeploymentList + ds, err = c.kube.AppsV1().Deployments(ns).List(ctx, listOption) + if err != nil { + err = fmt.Errorf("failed to list deployments in namespace %s: %w", ns, err) + return + } + deployments = append(deployments, ds.Items...) + + var dss *appsv1.DaemonSetList + dss, err = c.kube.AppsV1().DaemonSets(ns).List(ctx, listOption) + if err != nil { + err = fmt.Errorf("failed to list daemonsets in namespace %s: %w", ns, err) + return + } + daemonSets = append(daemonSets, dss.Items...) + + if len(ps.Items) > 0 || len(ds.Items) > 0 || len(dss.Items) > 0 { + distinctNamespaces = append(distinctNamespaces, ns) + } } - deployments = ds.Items - var dss *appsv1.DaemonSetList - dss, err = c.kube.AppsV1().DaemonSets("").List(ctx, listOption) - if err != nil { - err = fmt.Errorf("failed to list daemonsets: %w", err) + // All pods, deployments, and daemonsets should be in the same namespace. + // Otherwise, it would be a bug in the EG or the disruptive configuration change of EG. + if len(distinctNamespaces) > 1 { + err = fmt.Errorf("found gateway-labeled objects in multiple namespaces: %v", distinctNamespaces) return } - daemonSets = dss.Items - // We assume that all pods, deployments, and daemonsets are in the same namespace. Otherwise, it would be a bug in the EG - // or the disruptive configuration change of EG. if len(pods) > 0 { namespace = pods[0].Namespace } diff --git a/internal/controller/gateway_test.go b/internal/controller/gateway_test.go index 4cf4809acf..bd1063fd1a 100644 --- a/internal/controller/gateway_test.go +++ b/internal/controller/gateway_test.go @@ -53,7 +53,7 @@ func TestGatewayController_Reconcile(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) fakeKube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, fakeKube, ctrl.Log, + c := NewGatewayController(fakeClient, fakeKube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const namespace = "ns" @@ -175,7 +175,7 @@ func TestGatewayController_reconcileFilterConfigSecret(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -329,7 +329,7 @@ func TestGatewayController_reconcileFilterConfigSecret_HostnameScopedModels(t *t fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -425,7 +425,7 @@ func TestGatewayController_reconcileFilterConfigSecret_AllUnscopedRoutesLeaveUns fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -479,7 +479,7 @@ func TestGatewayController_reconcileFilterConfigSecret_RouteLevelLLMRequestCostA fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -580,7 +580,7 @@ func TestGatewayController_reconcileFilterConfigSecret_RouteLevelLLMRequestCostA fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -636,7 +636,7 @@ func TestGatewayController_reconcileFilterConfigSecret_InvalidCELExpression(t *t fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -675,7 +675,7 @@ func TestGatewayController_reconcileFilterConfigSecret_SkipsDeletedRoutes(t *tes fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -786,8 +786,7 @@ func TestGatewayController_bspToFilterAPIBackendAuth(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, - + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const namespace = "ns" @@ -987,7 +986,7 @@ func TestGatewayController_bspToFilterAPIBackendAuth(t *testing.T) { func TestGatewayController_bspToFilterAPIBackendAuth_ErrorCases(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) - c := NewGatewayController(fakeClient, fake2.NewClientset(), ctrl.Log, + c := NewGatewayController(fakeClient, fake2.NewClientset(), ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) ctx := context.Background() @@ -1048,7 +1047,7 @@ func TestGatewayController_bspToFilterAPIBackendAuth_ErrorCases(t *testing.T) { func TestGatewayController_GetSecretData_ErrorCases(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) - c := NewGatewayController(fakeClient, fake2.NewClientset(), ctrl.Log, + c := NewGatewayController(fakeClient, fake2.NewClientset(), ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) ctx := context.Background() @@ -1074,7 +1073,7 @@ func TestGatewayController_annotateGatewayPods(t *testing.T) { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) const v2Container = "ai-gateway-extproc:v2" const logLevel = "info" - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", v2Container, logLevel, false, nil, true) t.Run("pod with extproc", func(t *testing.T) { pod, err := kube.CoreV1().Pods(egNamespace).Create(t.Context(), &corev1.Pod{ @@ -1635,7 +1634,7 @@ func TestGatewayController_annotateDaemonSetGatewayPods(t *testing.T) { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) const v2Container = "ai-gateway-extproc:v2" const logLevel = "info" - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", v2Container, logLevel, false, nil, true) t.Run("pod without extproc", func(t *testing.T) { @@ -2048,7 +2047,7 @@ func TestGatewayController_backendWithMaybeBSP(t *testing.T) { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) const v2Container = "ai-gateway-extproc:v2" const logLevel = "info" - c := NewGatewayController(fakeClient, kube, ctrl.Log, v2Container, logLevel, false, nil, true) + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", v2Container, logLevel, false, nil, true) _, _, err := c.backendWithMaybeBSP(t.Context(), "foo", "bar") require.ErrorContains(t, err, `aiservicebackends.aigateway.envoyproxy.io "bar" not found`) @@ -2107,7 +2106,7 @@ func TestGatewayController_reconcileFilterMCPConfigSecret(t *testing.T) { fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -2599,7 +2598,7 @@ func TestGatewayController_reconcileFilterConfigSecret_GlobalDefaults(t *testing fakeClient := requireNewFakeClientWithIndexes(t) kube := fake2.NewClientset() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel}))) - c := NewGatewayController(fakeClient, kube, ctrl.Log, + c := NewGatewayController(fakeClient, kube, ctrl.Log, "envoy-gateway-system", "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) const gwNamespace = "ns" @@ -2812,3 +2811,29 @@ func Test_mergeBodyMutations(t *testing.T) { }) } } + +func TestGatewayController_getObjectsForGatewayNamespaceInconsistency(t *testing.T) { + const gwName, gwNamespace, egNamespace = "gw", "ns", "envoy-gateway-system" + labels := map[string]string{ + egOwningGatewayNameLabel: gwName, + egOwningGatewayNamespaceLabel: gwNamespace, + } + gw := &gwapiv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: gwName, Namespace: gwNamespace}} + + kube := fake2.NewClientset() + c := NewGatewayController(requireNewFakeClientWithIndexes(t), kube, ctrl.Log, egNamespace, + "docker.io/envoyproxy/ai-gateway-extproc:latest", "info", false, nil, true) + + // Place a pod in the gateway namespace and a pod in the envoy-gateway namespace so that + // objects are found in two distinct namespaces, which should trigger the error. + for _, ns := range []string{gwNamespace, egNamespace} { + _, err := kube.CoreV1().Pods(ns).Create(t.Context(), &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-" + ns, Namespace: ns, Labels: labels}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + + _, _, _, _, err := c.getObjectsForGateway(t.Context(), gw) + require.Error(t, err) + require.Contains(t, err.Error(), "found gateway-labeled objects in multiple namespaces") +} diff --git a/manifests/charts/ai-gateway-helm/templates/deployment.yaml b/manifests/charts/ai-gateway-helm/templates/deployment.yaml index 65556cdfd6..40862befdf 100644 --- a/manifests/charts/ai-gateway-helm/templates/deployment.yaml +++ b/manifests/charts/ai-gateway-helm/templates/deployment.yaml @@ -50,6 +50,7 @@ spec: - --extProcImage={{ .Values.extProc.image.repository }}:{{ .Values.extProc.image.tag | default .Chart.AppVersion }} - --extProcImagePullPolicy={{ .Values.extProc.imagePullPolicy }} - --extProcLogLevel={{ .Values.extProc.logLevel }} + - --envoyGatewayNamespace={{ .Values.envoyGateway.namespace }} {{- if .Values.extProc.enableRedaction }} - --extProcEnableRedaction=true {{- end }}