Skip to content

Commit 8921062

Browse files
Merge pull request #4000 from eslutsky/ingress-performance
USHIFT-4378: introduce ingress customization fields
2 parents cacaef4 + d66c874 commit 8921062

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+15422
-36
lines changed

assets/components/openshift-router/deployment.yaml

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ spec:
4040
- name: ROUTER_CIPHERSUITES
4141
value: TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
4242
- name: ROUTER_DISABLE_HTTP2
43-
value: "true"
43+
value: '{{.RouterDisableHttp2}}'
4444
- name: ROUTER_DISABLE_NAMESPACE_OWNERSHIP_CHECK
4545
value: '{{.RouterNamespaceOwnership}}'
4646
- name: ROUTER_LOAD_BALANCE_ALGORITHM
@@ -50,11 +50,11 @@ spec:
5050
- name: ROUTER_SERVICE_NAME
5151
value: default
5252
- name: ROUTER_SET_FORWARDED_HEADERS
53-
value: append
53+
value: '{{.ForwardedHeaderPolicy}}'
5454
- name: ROUTER_TCP_BALANCE_SCHEME
5555
value: source
5656
- name: ROUTER_THREADS
57-
value: "4"
57+
value: '{{.ThreadCount}}'
5858
- name: SSL_MIN_VERSION
5959
value: TLSv1.2
6060
- name: ROUTER_USE_PROXY_PROTOCOL
@@ -65,6 +65,34 @@ spec:
6565
value: apps.{{ .BaseDomain }}
6666
- name: ROUTER_IP_V4_V6_MODE
6767
value: '{{ .RouterMode }}'
68+
- name: ROUTER_BUF_SIZE
69+
value: '{{.RouterBufSize}}'
70+
- name: ROUTER_MAX_REWRITE_SIZE
71+
value: '{{.HeaderBufferMaxRewriteBytes}}'
72+
- name: ROUTER_BACKEND_CHECK_INTERVAL
73+
value: '{{.HealthCheckInterval}}'
74+
- name: ROUTER_DEFAULT_CLIENT_TIMEOUT
75+
value: '{{.ClientTimeout}}'
76+
- name: ROUTER_CLIENT_FIN_TIMEOUT
77+
value: '{{.ClientFinTimeout}}'
78+
- name: ROUTER_DEFAULT_SERVER_TIMEOUT
79+
value: '{{.ServerTimeout}}'
80+
- name: ROUTER_DEFAULT_SERVER_FIN_TIMEOUT
81+
value: '{{.ServerFinTimeout}}'
82+
- name: ROUTER_DEFAULT_TUNNEL_TIMEOUT
83+
value: '{{.TunnelTimeout}}'
84+
- name: ROUTER_INSPECT_DELAY
85+
value: '{{.TlsInspectDelay}}'
86+
- name: ROUTER_MAX_CONNECTIONS
87+
value: '{{.MaxConnections}}'
88+
- name: ROUTER_DONT_LOG_NULL
89+
value: '{{.LogEmptyRequests}}'
90+
- name: ROUTER_HTTP_IGNORE_PROBES
91+
value: '{{.HTTPEmptyRequestsPolicy}}'
92+
- name: ROUTER_ENABLE_COMPRESSION
93+
value: '{{.RouterEnableCompression}}'
94+
- name: ROUTER_COMPRESSION_MIME
95+
value: '{{.RouterCompressionMime}}'
6896
livenessProbe:
6997
failureThreshold: 3
7098
httpGet:

cmd/generate-config/config/config-openapi-spec.json

Lines changed: 117 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,16 +138,67 @@
138138
"listenAddress",
139139
"ports",
140140
"routeAdmissionPolicy",
141-
"status"
141+
"status",
142+
"tuningOptions"
142143
],
143144
"properties": {
145+
"defaultHTTPVersion": {
146+
"description": "Determines default http version should be used for the ingress backends\nBy default, using version 1.",
147+
"type": "integer",
148+
"format": "int32",
149+
"default": "1"
150+
},
151+
"forwardedHeaderPolicy": {
152+
"description": "forwardedHeaderPolicy specifies when and how ingress router\nsets the Forwarded, X-Forwarded-For, X-Forwarded-Host,\nX-Forwarded-Port, X-Forwarded-Proto, and X-Forwarded-Proto-Version\nHTTP headers. The value may be one of the following:\n\n\n* \"Append\", which specifies that ingress router appends the\n headers, preserving existing headers.\n\n\n* \"Replace\", which specifies that ingress router sets the\n headers, replacing any existing Forwarded or X-Forwarded-* headers.\n\n\n* \"IfNone\", which specifies that ingress router sets the\n headers if they are not already set.\n\n\n* \"Never\", which specifies that ingress router never sets the\n headers, preserving any existing headers.\n\n\nBy default, the policy is \"Append\".",
153+
"type": "string",
154+
"enum": [
155+
"Append",
156+
"Replace",
157+
"IfNone",
158+
"Never"
159+
]
160+
},
161+
"httpCompression": {
162+
"description": "httpCompression defines a policy for HTTP traffic compression.\nBy default, there is no HTTP compression.",
163+
"type": "object",
164+
"properties": {
165+
"mimeTypes": {
166+
"description": "mimeTypes is a list of MIME types that should have compression applied.\nThis list can be empty, in which case the ingress controller does not apply compression.\n\n\nNote: Not all MIME types benefit from compression, but HAProxy will still use resources\nto try to compress if instructed to. Generally speaking, text (html, css, js, etc.)\nformats benefit from compression, but formats that are already compressed (image,\naudio, video, etc.) benefit little in exchange for the time and cpu spent on compressing\nagain. See https://joehonton.medium.com/the-gzip-penalty-d31bd697f1a2",
167+
"type": "array",
168+
"items": {
169+
"description": "CompressionMIMEType defines the format of a single MIME type.\nE.g. \"text/css; charset=utf-8\", \"text/html\", \"text/*\", \"image/svg+xml\",\n\"application/octet-stream\", \"X-custom/customsub\", etc.\n\n\nThe format should follow the Content-Type definition in RFC 1341:\nContent-Type := type \"/\" subtype *[\";\" parameter]\n - The type in Content-Type can be one of:\n application, audio, image, message, multipart, text, video, or a custom\n type preceded by \"X-\" and followed by a token as defined below.\n - The token is a string of at least one character, and not containing white\n space, control characters, or any of the characters in the tspecials set.\n - The tspecials set contains the characters ()\u003c\u003e@,;:\\\"/[]?.=\n - The subtype in Content-Type is also a token.\n - The optional parameter/s following the subtype are defined as:\n token \"=\" (token / quoted-string)\n - The quoted-string, as defined in RFC 822, is surrounded by double quotes\n and can contain white space plus any character EXCEPT \\, \", and CR.\n It can also contain any single ASCII character as long as it is escaped by \\.",
170+
"type": "string",
171+
"pattern": "^(?i)(x-[^][ ()\\\\\u003c\u003e@,;:\"/?.=\\x00-\\x1F\\x7F]+|application|audio|image|message|multipart|text|video)/[^][ ()\\\\\u003c\u003e@,;:\"/?.=\\x00-\\x1F\\x7F]+(; *[^][ ()\\\\\u003c\u003e@,;:\"/?.=\\x00-\\x1F\\x7F]+=([^][ ()\\\\\u003c\u003e@,;:\"/?.=\\x00-\\x1F\\x7F]+|\"(\\\\[\\x00-\\x7F]|[^\\x0D\"\\\\])*\"))*$"
172+
},
173+
"x-kubernetes-list-type": "set"
174+
}
175+
}
176+
},
177+
"httpEmptyRequestsPolicy": {
178+
"description": "httpEmptyRequestsPolicy describes how HTTP connections should be\nhandled if the connection times out before a request is received.\nAllowed values for this field are \"Respond\" and \"Ignore\". If the\nfield is set to \"Respond\", the ingress controller sends an HTTP 400\nor 408 response, logs the connection (if access logging is enabled),\nand counts the connection in the appropriate metrics. If the field\nis set to \"Ignore\", the ingress controller closes the connection\nwithout sending a response, logging the connection, or incrementing\nmetrics. The default value is \"Respond\".\n\n\nTypically, these connections come from load balancers' health probes\nor Web browsers' speculative connections (\"preconnect\") and can be\nsafely ignored. However, these requests may also be caused by\nnetwork errors, and so setting this field to \"Ignore\" may impede\ndetection and diagnosis of problems. In addition, these requests may\nbe caused by port scans, in which case logging empty requests may aid\nin detecting intrusion attempts.",
179+
"type": "string",
180+
"default": "Respond",
181+
"enum": [
182+
"Respond",
183+
"Ignore"
184+
]
185+
},
144186
"listenAddress": {
145187
"description": "List of IP addresses and NIC names where the router will be listening. The NIC\nnames get translated to all their configured IPs dynamically. Defaults to the\nconfigured IPs in the host at MicroShift start.",
146188
"type": "array",
147189
"items": {
148190
"type": "string"
149191
}
150192
},
193+
"logEmptyRequests": {
194+
"description": "logEmptyRequests specifies how connections on which no request is\nreceived should be logged. Typically, these empty requests come from\nload balancers' health probes or Web browsers' speculative\nconnections (\"preconnect\"), in which case logging these requests may\nbe undesirable. However, these requests may also be caused by\nnetwork errors, in which case logging empty requests may be useful\nfor diagnosing the errors. In addition, these requests may be caused\nby port scans, in which case logging empty requests may aid in\ndetecting intrusion attempts. Allowed values for this field are\n\"Log\" and \"Ignore\". The default value is \"Log\".",
195+
"type": "string",
196+
"default": "Log",
197+
"enum": [
198+
"Log",
199+
"Ignore"
200+
]
201+
},
151202
"ports": {
152203
"type": "object",
153204
"required": [
@@ -184,6 +235,71 @@
184235
"description": "Default router status, can be Managed or Removed.",
185236
"type": "string",
186237
"default": "Managed"
238+
},
239+
"tuningOptions": {
240+
"description": "IngressControllerTuningOptions specifies options for tuning the performance\nof ingress controller pods",
241+
"type": "object",
242+
"properties": {
243+
"clientFinTimeout": {
244+
"description": "clientFinTimeout defines how long a connection will be held open while\nwaiting for the client response to the server/backend closing the\nconnection.\n\n\nIf unset, the default timeout is 1s",
245+
"type": "string",
246+
"format": "duration"
247+
},
248+
"clientTimeout": {
249+
"description": "clientTimeout defines how long a connection will be held open while\nwaiting for a client response.\n\n\nIf unset, the default timeout is 30s",
250+
"type": "string",
251+
"format": "duration"
252+
},
253+
"headerBufferBytes": {
254+
"description": "headerBufferBytes describes how much memory should be reserved\n(in bytes) for IngressController connection sessions.\nNote that this value must be at least 16384 if HTTP/2 is\nenabled for the IngressController (https://tools.ietf.org/html/rfc7540).\nIf this field is empty, the IngressController will use a default value\nof 32768 bytes.\n\n\nSetting this field is generally not recommended as headerBufferBytes\nvalues that are too small may break the IngressController and\nheaderBufferBytes values that are too large could cause the\nIngressController to use significantly more memory than necessary.",
255+
"type": "integer",
256+
"format": "int32",
257+
"minimum": 16384
258+
},
259+
"headerBufferMaxRewriteBytes": {
260+
"description": "headerBufferMaxRewriteBytes describes how much memory should be reserved\n(in bytes) from headerBufferBytes for HTTP header rewriting\nand appending for IngressController connection sessions.\nNote that incoming HTTP requests will be limited to\n(headerBufferBytes - headerBufferMaxRewriteBytes) bytes, meaning\nheaderBufferBytes must be greater than headerBufferMaxRewriteBytes.\nIf this field is empty, the IngressController will use a default value\nof 8192 bytes.\n\n\nSetting this field is generally not recommended as\nheaderBufferMaxRewriteBytes values that are too small may break the\nIngressController and headerBufferMaxRewriteBytes values that are too\nlarge could cause the IngressController to use significantly more memory\nthan necessary.",
261+
"type": "integer",
262+
"format": "int32",
263+
"minimum": 4096
264+
},
265+
"healthCheckInterval": {
266+
"description": "healthCheckInterval defines how long the router waits between two consecutive\nhealth checks on its configured backends. This value is applied globally as\na default for all routes, but may be overridden per-route by the route annotation\n\"router.openshift.io/haproxy.health.check.interval\".\n\n\nExpects an unsigned duration string of decimal numbers, each with optional\nfraction and a unit suffix, eg \"300ms\", \"1.5h\" or \"2h45m\".\nValid time units are \"ns\", \"us\" (or \"µs\" U+00B5 or \"μs\" U+03BC), \"ms\", \"s\", \"m\", \"h\".\n\n\nSetting this to less than 5s can cause excess traffic due to too frequent\nTCP health checks and accompanying SYN packet storms. Alternatively, setting\nthis too high can result in increased latency, due to backend servers that are no\nlonger available, but haven't yet been detected as such.\n\n\nAn empty or zero healthCheckInterval means no opinion and IngressController chooses\na default, which is subject to change over time.\nCurrently the default healthCheckInterval value is 5s.\n\n\nCurrently the minimum allowed value is 1s and the maximum allowed value is\n2147483647ms (24.85 days). Both are subject to change over time.",
267+
"type": "string",
268+
"pattern": "^(0|([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+)$"
269+
},
270+
"maxConnections": {
271+
"description": "maxConnections defines the maximum number of simultaneous\nconnections that can be established per HAProxy process.\nIncreasing this value allows each ingress controller pod to\nhandle more connections but at the cost of additional\nsystem resources being consumed.\n\n\nPermitted values are: empty, 0, -1, and the range\n2000-2000000.\n\n\nIf this field is empty or 0, the IngressController will use\nthe default value of 50000, but the default is subject to\nchange in future releases.\n\n\nIf the value is -1 then HAProxy will dynamically compute a\nmaximum value based on the available ulimits in the running\ncontainer. Selecting -1 (i.e., auto) will result in a large\nvalue being computed (~520000 on OpenShift \u003e=4.10 clusters)\nand therefore each HAProxy process will incur significant\nmemory usage compared to the current default of 50000.\n\n\nSetting a value that is greater than the current operating\nsystem limit will prevent the HAProxy process from\nstarting.\n\n\nIf you choose a discrete value (e.g., 750000) and the\nrouter pod is migrated to a new node, there's no guarantee\nthat that new node has identical ulimits configured. In\nsuch a scenario the pod would fail to start. If you have\nnodes with different ulimits configured (e.g., different\ntuned profiles) and you choose a discrete value then the\nguidance is to use -1 and let the value be computed\ndynamically at runtime.\n\n\nYou can monitor memory usage for router containers with the\nfollowing metric:\n'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'.\n\n\nYou can monitor memory usage of individual HAProxy\nprocesses in router containers with the following metric:\n'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'.",
272+
"type": "integer",
273+
"format": "int32"
274+
},
275+
"serverFinTimeout": {
276+
"description": "serverFinTimeout defines how long a connection will be held open while\nwaiting for the server/backend response to the client closing the\nconnection.\n\n\nIf unset, the default timeout is 1s",
277+
"type": "string",
278+
"format": "duration"
279+
},
280+
"serverTimeout": {
281+
"description": "serverTimeout defines how long a connection will be held open while\nwaiting for a server/backend response.\n\n\nIf unset, the default timeout is 30s",
282+
"type": "string",
283+
"format": "duration"
284+
},
285+
"threadCount": {
286+
"description": "threadCount defines the number of threads created per HAProxy process.\nCreating more threads allows each ingress controller pod to handle more\nconnections, at the cost of more system resources being used. HAProxy\ncurrently supports up to 64 threads. If this field is empty, the\nIngressController will use the default value. The current default is 4\nthreads, but this may change in future releases.\n\n\nSetting this field is generally not recommended. Increasing the number\nof HAProxy threads allows ingress controller pods to utilize more CPU\ntime under load, potentially starving other pods if set too high.\nReducing the number of threads may cause the ingress controller to\nperform poorly.",
287+
"type": "integer",
288+
"format": "int32",
289+
"maximum": 64,
290+
"minimum": 1
291+
},
292+
"tlsInspectDelay": {
293+
"description": "tlsInspectDelay defines how long the router can hold data to find a\nmatching route.\n\n\nSetting this too short can cause the router to fall back to the default\ncertificate for edge-terminated or reencrypt routes even when a better\nmatching certificate could be used.\n\n\nIf unset, the default inspect delay is 5s",
294+
"type": "string",
295+
"format": "duration"
296+
},
297+
"tunnelTimeout": {
298+
"description": "tunnelTimeout defines how long a tunnel connection (including\nwebsockets) will be held open while the tunnel is idle.\n\n\nIf unset, the default timeout is 1h",
299+
"type": "string",
300+
"format": "duration"
301+
}
302+
}
187303
}
188304
}
189305
},

docs/user/howto_config.md

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,33 @@ dns:
2828
etcd:
2929
memoryLimitMB: 0
3030
ingress:
31+
defaultHTTPVersion: 0
32+
forwardedHeaderPolicy: ""
33+
httpCompression:
34+
mimeTypes:
35+
- ""
36+
httpEmptyRequestsPolicy: ""
3137
listenAddress:
3238
- ""
39+
logEmptyRequests: ""
3340
ports:
3441
http: 0
3542
https: 0
3643
routeAdmissionPolicy:
3744
namespaceOwnership: ""
3845
status: ""
46+
tuningOptions:
47+
clientFinTimeout: ""
48+
clientTimeout: ""
49+
headerBufferBytes: 0
50+
headerBufferMaxRewriteBytes: 0
51+
healthCheckInterval: ""
52+
maxConnections: 0
53+
serverFinTimeout: ""
54+
serverTimeout: ""
55+
threadCount: 0
56+
tlsInspectDelay: ""
57+
tunnelTimeout: ""
3958
kubelet:
4059
manifests:
4160
kustomizePaths:
@@ -90,14 +109,33 @@ dns:
90109
etcd:
91110
memoryLimitMB: 0
92111
ingress:
112+
defaultHTTPVersion: 1
113+
forwardedHeaderPolicy: ""
114+
httpCompression:
115+
mimeTypes:
116+
- ""
117+
httpEmptyRequestsPolicy: Respond
93118
listenAddress:
94119
- ""
120+
logEmptyRequests: Log
95121
ports:
96122
http: 80
97123
https: 443
98124
routeAdmissionPolicy:
99125
namespaceOwnership: InterNamespaceAllowed
100126
status: Managed
127+
tuningOptions:
128+
clientFinTimeout: ""
129+
clientTimeout: ""
130+
headerBufferBytes: 0
131+
headerBufferMaxRewriteBytes: 0
132+
healthCheckInterval: ""
133+
maxConnections: 0
134+
serverFinTimeout: ""
135+
serverTimeout: ""
136+
threadCount: 0
137+
tlsInspectDelay: ""
138+
tunnelTimeout: ""
101139
kubelet:
102140
manifests:
103141
kustomizePaths:

etcd/vendor/github.com/openshift/api/operator/v1/Makefile

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

etcd/vendor/github.com/openshift/api/operator/v1/doc.go

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)