NE-1476: Added network policies for the router

manifests/00-cluster-role.yaml

@@ -155,6 +155,7 @@ rules:
   - networking.k8s.io
   resources:
   - ingressclasses
+  - networkpolicies
   verbs:
   - '*'
 

manifests/01-network-policy.yaml

@@ -0,0 +1,43 @@
+# We control the namespace, deny anything we do not explicitly allow
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-deny-all
+  namespace: openshift-ingress-operator
+  annotations:
+    capability.openshift.io/name: Ingress
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress
+---
+### Allow the operator needs to talk to the apiserver and dns,
+### but it also needs to be able to configure external DNS and the endpoints
+### are only known at run time.  So it needs wildcard egress.
+### Allow access to the metrics ports on the operators
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-allow
+  namespace: openshift-ingress-operator
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector:
+    matchLabels:
+      name: ingress-operator
+  policyTypes:
+  - Egress
+  - Ingress
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 9393

manifests/01-role.yaml

@@ -37,6 +37,13 @@ rules:
   - services
   verbs:
   - "*"
+
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - "*"
 ---
 # Role for the operator to delete Role and RoleBindings
 # in the openshift-config namespace.

pkg/manifests/assets/canary/networkpolicy-allow.yaml

@@ -0,0 +1,19 @@
+# Router Pods
+#  No egress
+#  Ingress to http on port 8888 and https on port 8443 (TCP)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+# name, namespace, labels and annotations are set at runtime
+spec:
+  podSelector:
+    # matchLabels are set at runtime
+    matchLabels: {}
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 8888
+    - protocol: TCP
+      port: 8443
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/canary/networkpolicy-deny-all.yaml

@@ -0,0 +1,11 @@
+# Default deny all policy for all pods in the namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: openshift-ingress-canary-deny-all
+  namespace: openshift-ingress-canary
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress

pkg/manifests/assets/gateway-api/gateway-networkpolicy-allow.yaml

@@ -0,0 +1,23 @@
+# Istio Pods for Gateway API
+#  Egress to api server and everything (routes set endpoints, so can be literally anything)
+#  Ingress need to be wildcarded as well since the ports are configurable
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: gateway-api-allow
+  namespace: openshift-ingress
+spec:
+  podSelector:
+    matchLabels:
+      istio.io/rev: "openshift-gateway"
+  ingress:
+  - from:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/router/networkpolicy-allow.yaml

@@ -0,0 +1,25 @@
+# Router Pods
+#  Egress to api server and everything (routes set endpoints, so can be literally anything)
+#  Ingress to http on port 80 and https on port 443 (TCP) and to metrics (1936)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+# name, namespace,labels and annotations are set at runtime
+spec:
+  podSelector:
+    # matchLabels are set at runtime
+    matchLabels: {}
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 80
+    - protocol: TCP
+      port: 443
+    - protocol: TCP
+      port: 1936
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/router/networkpolicy-deny-all.yaml

@@ -0,0 +1,11 @@
+# Default deny all policy for all pods in the namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+    name: openshift-ingress-deny-all
+    namespace: openshift-ingress
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress

pkg/manifests/manifests.go

@@ -13,6 +13,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
@@ -24,31 +25,36 @@ import (
 )
 
 const (
-	RouterNamespaceAsset          = "assets/router/namespace.yaml"
-	RouterServiceAccountAsset     = "assets/router/service-account.yaml"
-	RouterClusterRoleAsset        = "assets/router/cluster-role.yaml"
-	RouterClusterRoleBindingAsset = "assets/router/cluster-role-binding.yaml"
-	RouterDeploymentAsset         = "assets/router/deployment.yaml"
-	RouterServiceInternalAsset    = "assets/router/service-internal.yaml"
-	RouterServiceCloudAsset       = "assets/router/service-cloud.yaml"
+	RouterNamespaceAsset            = "assets/router/namespace.yaml"
+	RouterServiceAccountAsset       = "assets/router/service-account.yaml"
+	RouterClusterRoleAsset          = "assets/router/cluster-role.yaml"
+	RouterClusterRoleBindingAsset   = "assets/router/cluster-role-binding.yaml"
+	RouterDeploymentAsset           = "assets/router/deployment.yaml"
+	RouterServiceInternalAsset      = "assets/router/service-internal.yaml"
+	RouterServiceCloudAsset         = "assets/router/service-cloud.yaml"
+	RouterNetworkPolicyDenyAllAsset = "assets/router/networkpolicy-deny-all.yaml"
+	RouterNetworkPolicyAllowAsset   = "assets/router/networkpolicy-allow.yaml"
 
 	MetricsClusterRoleAsset        = "assets/router/metrics/cluster-role.yaml"
 	MetricsClusterRoleBindingAsset = "assets/router/metrics/cluster-role-binding.yaml"
 	MetricsRoleAsset               = "assets/router/metrics/role.yaml"
 	MetricsRoleBindingAsset        = "assets/router/metrics/role-binding.yaml"
 
-	CanaryNamespaceAsset = "assets/canary/namespace.yaml"
-	CanaryDaemonSetAsset = "assets/canary/daemonset.yaml"
-	CanaryServiceAsset   = "assets/canary/service.yaml"
-	CanaryRouteAsset     = "assets/canary/route.yaml"
-
-	GatewayClassCRDAsset            = "assets/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml"
-	GatewayCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_gateways.yaml"
-	GRPCRouteCRDAsset               = "assets/gateway-api/gateway.networking.k8s.io_grpcroutes.yaml"
-	HTTPRouteCRDAsset               = "assets/gateway-api/gateway.networking.k8s.io_httproutes.yaml"
-	ReferenceGrantCRDAsset          = "assets/gateway-api/gateway.networking.k8s.io_referencegrants.yaml"
-	GatewayAPIAdminClusterRoleAsset = "assets/gateway-api/aggregated-cluster-roles/admin-cluster-role.yaml"
-	GatewayAPIViewClusterRoleAsset  = "assets/gateway-api/aggregated-cluster-roles/view-cluster-role.yaml"
+	CanaryNamespaceAsset            = "assets/canary/namespace.yaml"
+	CanaryDaemonSetAsset            = "assets/canary/daemonset.yaml"
+	CanaryServiceAsset              = "assets/canary/service.yaml"
+	CanaryRouteAsset                = "assets/canary/route.yaml"
+	CanaryNetworkPolicyDenyAllAsset = "assets/canary/networkpolicy-deny-all.yaml"
+	CanaryNetworkPolicyAllowAsset   = "assets/canary/networkpolicy-allow.yaml"
+
+	GatewayClassCRDAsset              = "assets/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml"
+	GatewayCRDAsset                   = "assets/gateway-api/gateway.networking.k8s.io_gateways.yaml"
+	GRPCRouteCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_grpcroutes.yaml"
+	HTTPRouteCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_httproutes.yaml"
+	ReferenceGrantCRDAsset            = "assets/gateway-api/gateway.networking.k8s.io_referencegrants.yaml"
+	GatewayAPIAdminClusterRoleAsset   = "assets/gateway-api/aggregated-cluster-roles/admin-cluster-role.yaml"
+	GatewayAPIViewClusterRoleAsset    = "assets/gateway-api/aggregated-cluster-roles/view-cluster-role.yaml"
+	GatewayAPIAllowNetworkPolicyAsset = "assets/gateway-api/gateway-networkpolicy-allow.yaml"
 
 	// Annotation used to inform the certificate generation service to
 	// generate a cluster-signed certificate and populate the secret.
@@ -258,6 +264,22 @@ func CanaryRoute() *routev1.Route {
 	return route
 }
 
+func CanaryNetworkPolicyDenyAll() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(CanaryNetworkPolicyDenyAllAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func CanaryNetworkPolicyAllow() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(CanaryNetworkPolicyAllowAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
 func GatewayClassCRD() *apiextensionsv1.CustomResourceDefinition {
 	crd, err := NewCustomResourceDefinition(MustAssetReader(GatewayClassCRDAsset))
 	if err != nil {
@@ -314,6 +336,30 @@ func GatewayAPIViewClusterRole() *rbacv1.ClusterRole {
 	return clusterRole
 }
 
+func GatewayAPIAllowNetworkPolicy() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(GatewayAPIAllowNetworkPolicyAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func RouterNetworkPolicyDenyAll() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyDenyAllAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func RouterNetworkPolicyAllow() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyAllowAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
 func NewServiceAccount(manifest io.Reader) (*corev1.ServiceAccount, error) {
 	sa := corev1.ServiceAccount{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&sa); err != nil {
@@ -404,6 +450,15 @@ func NewRoute(manifest io.Reader) (*routev1.Route, error) {
 	return &o, nil
 }
 
+func NewNetworkPolicy(manifest io.Reader) (*networkingv1.NetworkPolicy, error) {
+	o := networkingv1.NetworkPolicy{}
+	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {
+		return nil, err
+	}
+
+	return &o, nil
+}
+
 func NewCustomResourceDefinition(manifest io.Reader) (*apiextensionsv1.CustomResourceDefinition, error) {
 	o := apiextensionsv1.CustomResourceDefinition{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {

pkg/manifests/manifests_test.go

@@ -35,7 +35,8 @@ func TestManifests(t *testing.T) {
 	RouterClusterRole()
 	RouterClusterRoleBinding()
 	RouterStatsSecret(ci)
-
+	RouterNetworkPolicyDenyAll()
+	RouterNetworkPolicyAllow()
 	MetricsClusterRole()
 	MetricsClusterRoleBinding()
 	MetricsRole()
@@ -51,6 +52,14 @@ func TestManifests(t *testing.T) {
 	GRPCRouteCRD()
 	HTTPRouteCRD()
 	ReferenceGrantCRD()
+	GatewayAPIAllowNetworkPolicy()
+
+	CanaryNamespace()
+	CanaryDaemonSet()
+	CanaryService()
+	CanaryRoute()
+	CanaryNetworkPolicyDenyAll()
+	CanaryNetworkPolicyAllow()
 
 	adminOnlyVerbs := []string{"create", "update", "patch", "delete", "deletecollection"}
 	GatewayAPIAdminClusterRole()

pkg/operator/controller/canary/controller.go

@@ -22,6 +22,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -160,7 +161,13 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
 	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Service{}, enqueueRequestForDefaultIngressController(config.Namespace), canaryServicePredicate)); err != nil {
 		return nil, err
 	}
-
+	canaryNetworkPolicyPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
+		canaryNetworkPolicy := operatorcontroller.CanaryNetworkPolicyName()
+		return o.GetNamespace() == canaryNetworkPolicy.Namespace && o.GetName() == canaryNetworkPolicy.Name
+	})
+	if err := c.Watch(source.Kind[client.Object](operatorCache, &networkingv1.NetworkPolicy{}, enqueueRequestForDefaultIngressController(config.Namespace), canaryNetworkPolicyPredicate)); err != nil {
+		return nil, err
+	}
 	return c, nil
 }
 
@@ -189,6 +196,14 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
 		return result, fmt.Errorf("failed to ensure canary namespace: %v", err)
 	}
 
+	if _, _, err := r.ensureCanaryNamespaceNetworkPolicy(); err != nil {
+		return result, fmt.Errorf("failed to ensure canary namespace network policy: %v", err)
+	}
+
+	if _, _, err := r.ensureCanaryNetworkPolicy(); err != nil {
+		return result, fmt.Errorf("failed to ensure canary network policy: %v", err)
+	}
+
 	haveDs, daemonset, err := r.ensureCanaryDaemonSet()
 	if err != nil {
 		return result, fmt.Errorf("failed to ensure canary daemonset: %v", err)

pkg/operator/controller/canary/namespace.go

@@ -8,6 +8,7 @@ import (
 	"github.com/openshift/cluster-ingress-operator/pkg/operator/controller"
 
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 
@@ -94,3 +95,17 @@ func canaryNamespaceChanged(current, expected *corev1.Namespace) (bool, *corev1.
 
 	return true, updated
 }
+
+func (r *reconciler) ensureCanaryNamespaceNetworkPolicy() (bool, *networkingv1.NetworkPolicy, error) {
+	np := manifests.CanaryNetworkPolicyDenyAll()
+	if err := r.client.Get(context.TODO(), types.NamespacedName{Namespace: np.Namespace, Name: np.Name}, np); err != nil {
+		if !errors.IsNotFound(err) {
+			return false, nil, fmt.Errorf("failed to get canary namespace network policy %s/%s: %v", np.Namespace, np.Name, err)
+		}
+		if err := r.client.Create(context.TODO(), np); err != nil {
+			return false, nil, fmt.Errorf("failed to create canary namespace network policy %s/%s: %v", np.Namespace, np.Name, err)
+		}
+		log.Info("created canary namespace network policy", "namespace", np.Namespace, "name", np.Name)
+	}
+	return true, np, nil
+}