Backport of add gateway failures into release/2.0.x (#23569)

backport of commit 8af3ade

Co-authored-by: Sanika Chavan <sanika.vikaschavan@hashicorp.com>

Author: hc-github-team-consul-core

.changelog/23563.txt

@@ -0,0 +1 @@
+xds: add `Consecutive5xx`, `ConsecutiveGatewayFailure`, and `EnforcingConsecutiveGatewayFailure` fields to `PassiveHealthCheck`, allowing operators to configure Envoy outlier detection thresholds for 5xx responses and gateway failures (502/503/504) on upstreams defaults.

agent/consul/config_endpoint_test.go

@@ -1684,11 +1684,14 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 						Upstream: wildcard,
 						Config: map[string]interface{}{
 							"passive_health_check": map[string]interface{}{
-								"Interval":                int64(10),
-								"MaxFailures":             int64(2),
-								"EnforcingConsecutive5xx": int64(60),
-								"MaxEjectionPercent":      int64(61),
-								"BaseEjectionTime":        uint64(62 * time.Second),
+								"Interval":                           int64(10),
+								"MaxFailures":                        int64(2),
+								"EnforcingConsecutive5xx":            int64(60),
+								"EnforcingConsecutiveGatewayFailure": interface{}(nil),
+								"Consecutive5xx":                     interface{}(nil),
+								"ConsecutiveGatewayFailure":          interface{}(nil),
+								"MaxEjectionPercent":                 int64(61),
+								"BaseEjectionTime":                   uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "none",
@@ -1700,11 +1703,14 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 						Upstream: mysql,
 						Config: map[string]interface{}{
 							"passive_health_check": map[string]interface{}{
-								"Interval":                int64(10),
-								"MaxFailures":             int64(2),
-								"EnforcingConsecutive5xx": int64(60),
-								"MaxEjectionPercent":      int64(61),
-								"BaseEjectionTime":        uint64(62 * time.Second),
+								"Interval":                           int64(10),
+								"MaxFailures":                        int64(2),
+								"EnforcingConsecutive5xx":            int64(60),
+								"EnforcingConsecutiveGatewayFailure": interface{}(nil),
+								"Consecutive5xx":                     interface{}(nil),
+								"ConsecutiveGatewayFailure":          interface{}(nil),
+								"MaxEjectionPercent":                 int64(61),
+								"BaseEjectionTime":                   uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "local",

agent/proxycfg/proxycfg.deepcopy.go

@@ -1209,6 +1209,19 @@ type PassiveHealthCheck struct {
 	// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100.
 	EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
 
+	// EnforcingConsecutiveGatewayFailure is the % chance that a host will be actually ejected
+	// when an outlier status is detected through consecutive gateway failures.
+	// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 0.
+	EnforcingConsecutiveGatewayFailure *uint32 `json:",omitempty" alias:"enforcing_consecutive_gateway_failure"`
+
+	// Consecutive5xx is the number of consecutive 5xx responses that trigger outlier detection.
+	// If not set, defaults to 5. Setting this overrides MaxFailures for 5xx detection.
+	Consecutive5xx *uint32 `json:",omitempty" alias:"consecutive_5xx"`
+
+	// ConsecutiveGatewayFailure is the number of consecutive gateway failures (502, 503, 504)
+	// that trigger outlier detection. If not set, defaults to 0 (disabled).
+	ConsecutiveGatewayFailure *uint32 `json:",omitempty" alias:"consecutive_gateway_failure"`
+
 	// The maximum % of an upstream cluster that can be ejected due to outlier detection.
 	// Defaults to 10% but will eject at least one host regardless of the value.
 	// TODO: remove me
@@ -1240,6 +1253,15 @@ func (chk PassiveHealthCheck) Validate() error {
 	if chk.EnforcingConsecutive5xx != nil && *chk.EnforcingConsecutive5xx > 100 {
 		return fmt.Errorf("passive health check enforcing_consecutive_5xx must be a percentage between 0 and 100")
 	}
+	if chk.EnforcingConsecutiveGatewayFailure != nil && *chk.EnforcingConsecutiveGatewayFailure > 100 {
+		return fmt.Errorf("passive health check enforcing_consecutive_gateway_failure must be a percentage between 0 and 100")
+	}
+	if chk.Consecutive5xx != nil && *chk.Consecutive5xx < 1 {
+		return fmt.Errorf("passive health check consecutive_5xx must be greater than 0")
+	}
+	if chk.ConsecutiveGatewayFailure != nil && *chk.ConsecutiveGatewayFailure < 1 {
+		return fmt.Errorf("passive health check consecutive_gateway_failure must be greater than 0")
+	}
 	if chk.MaxEjectionPercent != nil && *chk.MaxEjectionPercent > 100 {
 		return fmt.Errorf("passive health check max_ejection_percent must be a percentage between 0 and 100")
 	}

agent/structs/config_entry.go

@@ -8,9 +8,8 @@ import (
 	"time"
 
 	envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
-	"google.golang.org/protobuf/types/known/durationpb"
-
 	"github.com/go-viper/mapstructure/v2"
+	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	"github.com/hashicorp/consul/agent/structs"
@@ -215,13 +214,22 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
 			od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: p.MaxFailures}
 		}
 
-		if p.EnforcingConsecutive5xx != nil {
+		if p.Consecutive5xx != nil && *p.Consecutive5xx != 0 {
+			od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.Consecutive5xx}
+		}
+
+		if p.ConsecutiveGatewayFailure != nil && *p.ConsecutiveGatewayFailure != 0 {
+			od.ConsecutiveGatewayFailure = &wrapperspb.UInt32Value{Value: *p.ConsecutiveGatewayFailure}
+		}
+
+		if p.EnforcingConsecutive5xx != nil && (*p.EnforcingConsecutive5xx != 0 || allowZero) {
 			// NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway
-			if *p.EnforcingConsecutive5xx != 0 {
-				od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
-			} else if allowZero {
-				od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
-			}
+			od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
+		}
+
+		if p.EnforcingConsecutiveGatewayFailure != nil && (*p.EnforcingConsecutiveGatewayFailure != 0 || allowZero) {
+			// NOTE: EnforcingConsecutiveGatewayFailure must be greater than 0 for ingress-gateway
+			od.EnforcingConsecutiveGatewayFailure = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutiveGatewayFailure}
 		}
 
 		if p.MaxEjectionPercent != nil {
@@ -244,11 +252,24 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
 		od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: override.MaxFailures}
 	}
 
-	if override.EnforcingConsecutive5xx != nil {
+	if override.Consecutive5xx != nil && *override.Consecutive5xx != 0 {
+		od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.Consecutive5xx}
+	}
+
+	if override.ConsecutiveGatewayFailure != nil && *override.ConsecutiveGatewayFailure != 0 {
+		od.ConsecutiveGatewayFailure = &wrapperspb.UInt32Value{Value: *override.ConsecutiveGatewayFailure}
+	}
+
+	if override.EnforcingConsecutive5xx != nil && *override.EnforcingConsecutive5xx != 0 {
 		// NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway
-		if *override.EnforcingConsecutive5xx != 0 {
-			od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutive5xx}
-		}
+		od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutive5xx}
+		// Because only ingress gateways have overrides and they cannot have a value of 0, there is no allowZero
+		// override case to handle
+	}
+
+	if override.EnforcingConsecutiveGatewayFailure != nil && *override.EnforcingConsecutiveGatewayFailure != 0 {
+		// NOTE: EnforcingConsecutiveGatewayFailure must be greater than 0 for ingress-gateway
+		od.EnforcingConsecutiveGatewayFailure = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutiveGatewayFailure}
 		// Because only ingress gateways have overrides and they cannot have a value of 0, there is no allowZero
 		// override case to handle
 	}

agent/structs/structs.deepcopy.go

@@ -288,6 +288,19 @@ type PassiveHealthCheck struct {
 	// This setting can be used to disable ejection or to ramp it up slowly.
 	EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
 
+	// EnforcingConsecutiveGatewayFailure is the % chance that a host will be actually ejected
+	// when an outlier status is detected through consecutive gateway failures(codes-502, 503, 504).
+	// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 0.
+	EnforcingConsecutiveGatewayFailure *uint32 `json:",omitempty" alias:"enforcing_consecutive_gateway_failure"`
+
+	// Consecutive5xx is the number of consecutive 5xx responses that trigger outlier detection.
+	// If not set, defaults to 5. Setting this overrides MaxFailures for 5xx detection.
+	Consecutive5xx *uint32 `json:",omitempty" alias:"consecutive_5xx"`
+
+	// ConsecutiveGatewayFailure is the number of consecutive gateway failures (502, 503, 504)
+	// that trigger outlier detection. If not set, defaults to 0 (disabled).
+	ConsecutiveGatewayFailure *uint32 `json:",omitempty" alias:"consecutive_gateway_failure"`
+
 	// The maximum % of an upstream cluster that can be ejected due to outlier detection.
 	// Defaults to 10% but will eject at least one host regardless of the value.
 	MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`