From 90df9167969449daa955b8c659ab45c78e1a3125 Mon Sep 17 00:00:00 2001 From: Andrey Lebedev Date: Mon, 18 Nov 2024 10:54:38 +0000 Subject: [PATCH] OCPBUGS-48377: Add IdleCloseOnResponse field to IngressControllerSpec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an `IdleConnectionTerminationPolicy` field to control whether HAProxy keeps idle frontend connections open during a soft stop (router reload). Allow users to prevent errors in clients or load balancers that do not properly handle connection resets. This commit is based on https://github.com/openshift/api/pull/2102 with a key in the default value. For OCP versions <= 4.18, the default is `Deferred`, aligning with older releases where HAProxy’s `idle-close-on-response` option was added unconditionally. --- .../generated_openapi/zz_generated.openapi.go | 8 ++ openapi/openapi.json | 5 ++ .../AAA_ungated.yaml | 5 ++ .../IngressControllerLBOpenStack.yaml | 3 + .../IngressControllerLBSubnetsAWS.yaml | 6 ++ .../SetEIPForNLBIngressController.yaml | 9 +- operator/v1/types_ingress.go | 89 +++++++++++++++++++ ..._50_ingress_00_ingresscontrollers.crd.yaml | 70 +++++++++++++++ .../AAA_ungated.yaml | 70 +++++++++++++++ .../IngressControllerLBSubnetsAWS.yaml | 70 +++++++++++++++ .../SetEIPForNLBIngressController.yaml | 70 +++++++++++++++ .../v1/zz_generated.swagger_doc_generated.go | 37 ++++---- 12 files changed, 423 insertions(+), 19 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 18c91c0dd11..fdad1605215 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -48896,6 +48896,14 @@ func schema_openshift_api_operator_v1_IngressControllerSpec(ref common.Reference Ref: ref("github.com/openshift/api/operator/v1.HTTPCompressionPolicy"), }, }, + "idleConnectionTerminationPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Deferred\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", + Default: "Deferred", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index e8d65bf4fee..035f739c8d4 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -28418,6 +28418,11 @@ "description": "httpHeaders defines policy for HTTP headers.\n\nIf this field is empty, the default values are used.", "$ref": "#/definitions/com.github.openshift.api.operator.v1.IngressControllerHTTPHeaders" }, + "idleConnectionTerminationPolicy": { + "description": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Deferred\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", + "type": "string", + "default": "Deferred" + }, "logging": { "description": "logging defines parameters for what should be logged where. If this field is empty, operational logs are enabled but access logs are disabled.", "$ref": "#/definitions/com.github.openshift.api.operator.v1.IngressControllerLogging" diff --git a/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml b/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml index e1fd61edc67..ba3316d5450 100644 --- a/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1/tests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml @@ -15,6 +15,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred - name: Should be able to create an IngressController with valid Actions initial: | apiVersion: operator.openshift.io/v1 @@ -75,6 +76,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred httpHeaders: actions: response: @@ -496,6 +498,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred tuningOptions: connectTimeout: 10s - name: Should be able to create an IngressController with valid composite connect timeout @@ -516,6 +519,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred tuningOptions: connectTimeout: 100ms300μs - name: Should be able to create an IngressController with valid fraction connect timeout @@ -536,6 +540,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred tuningOptions: connectTimeout: 1.5m - name: Should not be able to create an IngressController with invalid unit connect timeout diff --git a/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBOpenStack.yaml b/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBOpenStack.yaml index 0945413ab6e..9c9d08754b1 100644 --- a/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBOpenStack.yaml +++ b/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBOpenStack.yaml @@ -19,6 +19,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -42,6 +43,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -67,6 +69,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: diff --git a/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml b/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml index 9c418f01014..eab0420d3be 100644 --- a/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml +++ b/operator/v1/tests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml @@ -30,6 +30,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -71,6 +72,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -184,6 +186,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -474,6 +477,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -531,6 +535,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: @@ -580,6 +585,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: type: LoadBalancerService loadBalancer: diff --git a/operator/v1/tests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml b/operator/v1/tests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml index a3020ff56e4..b9eaf0a2885 100644 --- a/operator/v1/tests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml +++ b/operator/v1/tests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml @@ -15,6 +15,7 @@ tests: kind: IngressController spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred - name: Should allow to set NLB parameters when LBType is NLB. initial: | apiVersion: operator.openshift.io/v1 @@ -43,6 +44,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -379,6 +381,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -433,6 +436,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -486,6 +490,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -534,6 +539,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -555,6 +561,7 @@ tests: namespace: openshift-ingress-operator spec: httpEmptyRequestsPolicy: Respond + idleConnectionTerminationPolicy: Deferred endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Managed @@ -567,4 +574,4 @@ tests: eipAllocations: - eipalloc-1234567890abcdefa - eipalloc-1234567890abcdefb - type: LoadBalancerService \ No newline at end of file + type: LoadBalancerService diff --git a/operator/v1/types_ingress.go b/operator/v1/types_ingress.go index 1f566434564..61c0bcdbb41 100644 --- a/operator/v1/types_ingress.go +++ b/operator/v1/types_ingress.go @@ -258,6 +258,75 @@ type IngressControllerSpec struct { // // +optional HTTPCompression HTTPCompressionPolicy `json:"httpCompression,omitempty"` + + // idleConnectionTerminationPolicy maps directly to HAProxy's + // idle-close-on-response option and controls whether HAProxy + // keeps idle frontend connections open during a soft stop + // (router reload). + // + // Allowed values for this field are "Immediate" and + // "Deferred". The default value is "Deferred". + // + // When set to "Immediate", idle connections are closed + // immediately during router reloads. This ensures immediate + // propagation of route changes but may impact clients + // sensitive to connection resets. + // + // When set to "Deferred", HAProxy will maintain idle + // connections during a soft reload instead of closing them + // immediately. These connections remain open until any of the + // following occurs: + // + // - A new request is received on the connection, in which + // case HAProxy handles it in the old process and closes + // the connection after sending the response. + // + // - HAProxy's `timeout http-keep-alive` duration expires + // (300 seconds in OpenShift's configuration, not + // configurable). + // + // - The client's keep-alive timeout expires, causing the + // client to close the connection. + // + // Setting Deferred can help prevent errors in clients or load + // balancers that do not properly handle connection resets. + // Additionally, this option allows you to retain the pre-2.4 + // HAProxy behaviour: in HAProxy version 2.2 (OpenShift + // versions < 4.14), maintaining idle connections during a + // soft reload was the default behaviour, but starting with + // HAProxy 2.4, the default changed to closing idle + // connections immediately. + // + // Important Consideration: + // + // - Using Deferred will result in temporary inconsistencies + // for the first request on each persistent connection + // after a route update and router reload. This request + // will be processed by the old HAProxy process using its + // old configuration. Subsequent requests will use the + // updated configuration. + // + // Operational Considerations: + // + // - Keeping idle connections open during reloads may lead + // to an accumulation of old HAProxy processes if + // connections remain idle for extended periods, + // especially in environments where frequent reloads + // occur. + // + // - Consider monitoring the number of HAProxy processes in + // the router pods when Deferred is set. + // + // - You may need to enable or adjust the + // `ingress.operator.openshift.io/hard-stop-after` + // duration (configured via an annotation on the + // IngressController resource) in environments with + // frequent reloads to prevent resource exhaustion. + // + // +optional + // +kubebuilder:default:="Deferred" + // +default="Deferred" + IdleConnectionTerminationPolicy IngressControllerConnectionTerminationPolicy `json:"idleConnectionTerminationPolicy,omitempty"` } // httpCompressionPolicy turns on compression for the specified MIME types. @@ -2068,3 +2137,23 @@ type IngressControllerList struct { Items []IngressController `json:"items"` } + +// IngressControllerConnectionTerminationPolicy defines the behaviour +// for handling idle connections during a soft reload of the router. +// +// +kubebuilder:validation:Enum=Immediate;Deferred +type IngressControllerConnectionTerminationPolicy string + +const ( + // IngressControllerConnectionTerminationPolicyImmediate specifies + // that idle connections should be closed immediately during a + // router reload. + IngressControllerConnectionTerminationPolicyImmediate IngressControllerConnectionTerminationPolicy = "Immediate" + + // IngressControllerConnectionTerminationPolicyDeferred + // specifies that idle connections should remain open until a + // terminating event, such as a new request, the expiration of + // the proxy keep-alive timeout, or the client closing the + // connection. + IngressControllerConnectionTerminationPolicyDeferred IngressControllerConnectionTerminationPolicy = "Deferred" +) diff --git a/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml b/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml index 2524b5a84a2..11fb0cc9c74 100644 --- a/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml +++ b/operator/v1/zz_generated.crd-manifests/0000_50_ingress_00_ingresscontrollers.crd.yaml @@ -1255,6 +1255,76 @@ spec: type: string type: object type: object + idleConnectionTerminationPolicy: + default: Deferred + description: |- + idleConnectionTerminationPolicy maps directly to HAProxy's + idle-close-on-response option and controls whether HAProxy + keeps idle frontend connections open during a soft stop + (router reload). + + Allowed values for this field are "Immediate" and + "Deferred". The default value is "Deferred". + + When set to "Immediate", idle connections are closed + immediately during router reloads. This ensures immediate + propagation of route changes but may impact clients + sensitive to connection resets. + + When set to "Deferred", HAProxy will maintain idle + connections during a soft reload instead of closing them + immediately. These connections remain open until any of the + following occurs: + + - A new request is received on the connection, in which + case HAProxy handles it in the old process and closes + the connection after sending the response. + + - HAProxy's `timeout http-keep-alive` duration expires + (300 seconds in OpenShift's configuration, not + configurable). + + - The client's keep-alive timeout expires, causing the + client to close the connection. + + Setting Deferred can help prevent errors in clients or load + balancers that do not properly handle connection resets. + Additionally, this option allows you to retain the pre-2.4 + HAProxy behaviour: in HAProxy version 2.2 (OpenShift + versions < 4.14), maintaining idle connections during a + soft reload was the default behaviour, but starting with + HAProxy 2.4, the default changed to closing idle + connections immediately. + + Important Consideration: + + - Using Deferred will result in temporary inconsistencies + for the first request on each persistent connection + after a route update and router reload. This request + will be processed by the old HAProxy process using its + old configuration. Subsequent requests will use the + updated configuration. + + Operational Considerations: + + - Keeping idle connections open during reloads may lead + to an accumulation of old HAProxy processes if + connections remain idle for extended periods, + especially in environments where frequent reloads + occur. + + - Consider monitoring the number of HAProxy processes in + the router pods when Deferred is set. + + - You may need to enable or adjust the + `ingress.operator.openshift.io/hard-stop-after` + duration (configured via an annotation on the + IngressController resource) in environments with + frequent reloads to prevent resource exhaustion. + enum: + - Immediate + - Deferred + type: string logging: description: |- logging defines parameters for what should be logged where. If this diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml index a0caca44a9c..8f19fe482f1 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/AAA_ungated.yaml @@ -1049,6 +1049,76 @@ spec: type: string type: object type: object + idleConnectionTerminationPolicy: + default: Deferred + description: |- + idleConnectionTerminationPolicy maps directly to HAProxy's + idle-close-on-response option and controls whether HAProxy + keeps idle frontend connections open during a soft stop + (router reload). + + Allowed values for this field are "Immediate" and + "Deferred". The default value is "Deferred". + + When set to "Immediate", idle connections are closed + immediately during router reloads. This ensures immediate + propagation of route changes but may impact clients + sensitive to connection resets. + + When set to "Deferred", HAProxy will maintain idle + connections during a soft reload instead of closing them + immediately. These connections remain open until any of the + following occurs: + + - A new request is received on the connection, in which + case HAProxy handles it in the old process and closes + the connection after sending the response. + + - HAProxy's `timeout http-keep-alive` duration expires + (300 seconds in OpenShift's configuration, not + configurable). + + - The client's keep-alive timeout expires, causing the + client to close the connection. + + Setting Deferred can help prevent errors in clients or load + balancers that do not properly handle connection resets. + Additionally, this option allows you to retain the pre-2.4 + HAProxy behaviour: in HAProxy version 2.2 (OpenShift + versions < 4.14), maintaining idle connections during a + soft reload was the default behaviour, but starting with + HAProxy 2.4, the default changed to closing idle + connections immediately. + + Important Consideration: + + - Using Deferred will result in temporary inconsistencies + for the first request on each persistent connection + after a route update and router reload. This request + will be processed by the old HAProxy process using its + old configuration. Subsequent requests will use the + updated configuration. + + Operational Considerations: + + - Keeping idle connections open during reloads may lead + to an accumulation of old HAProxy processes if + connections remain idle for extended periods, + especially in environments where frequent reloads + occur. + + - Consider monitoring the number of HAProxy processes in + the router pods when Deferred is set. + + - You may need to enable or adjust the + `ingress.operator.openshift.io/hard-stop-after` + duration (configured via an annotation on the + IngressController resource) in environments with + frequent reloads to prevent resource exhaustion. + enum: + - Immediate + - Deferred + type: string logging: description: |- logging defines parameters for what should be logged where. If this diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml index be89438d6a5..f8a916b093b 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/IngressControllerLBSubnetsAWS.yaml @@ -1198,6 +1198,76 @@ spec: type: string type: object type: object + idleConnectionTerminationPolicy: + default: Deferred + description: |- + idleConnectionTerminationPolicy maps directly to HAProxy's + idle-close-on-response option and controls whether HAProxy + keeps idle frontend connections open during a soft stop + (router reload). + + Allowed values for this field are "Immediate" and + "Deferred". The default value is "Deferred". + + When set to "Immediate", idle connections are closed + immediately during router reloads. This ensures immediate + propagation of route changes but may impact clients + sensitive to connection resets. + + When set to "Deferred", HAProxy will maintain idle + connections during a soft reload instead of closing them + immediately. These connections remain open until any of the + following occurs: + + - A new request is received on the connection, in which + case HAProxy handles it in the old process and closes + the connection after sending the response. + + - HAProxy's `timeout http-keep-alive` duration expires + (300 seconds in OpenShift's configuration, not + configurable). + + - The client's keep-alive timeout expires, causing the + client to close the connection. + + Setting Deferred can help prevent errors in clients or load + balancers that do not properly handle connection resets. + Additionally, this option allows you to retain the pre-2.4 + HAProxy behaviour: in HAProxy version 2.2 (OpenShift + versions < 4.14), maintaining idle connections during a + soft reload was the default behaviour, but starting with + HAProxy 2.4, the default changed to closing idle + connections immediately. + + Important Consideration: + + - Using Deferred will result in temporary inconsistencies + for the first request on each persistent connection + after a route update and router reload. This request + will be processed by the old HAProxy process using its + old configuration. Subsequent requests will use the + updated configuration. + + Operational Considerations: + + - Keeping idle connections open during reloads may lead + to an accumulation of old HAProxy processes if + connections remain idle for extended periods, + especially in environments where frequent reloads + occur. + + - Consider monitoring the number of HAProxy processes in + the router pods when Deferred is set. + + - You may need to enable or adjust the + `ingress.operator.openshift.io/hard-stop-after` + duration (configured via an annotation on the + IngressController resource) in environments with + frequent reloads to prevent resource exhaustion. + enum: + - Immediate + - Deferred + type: string logging: description: |- logging defines parameters for what should be logged where. If this diff --git a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml index c9093ecf8e0..3bd087d13cb 100644 --- a/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml +++ b/operator/v1/zz_generated.featuregated-crd-manifests/ingresscontrollers.operator.openshift.io/SetEIPForNLBIngressController.yaml @@ -1108,6 +1108,76 @@ spec: type: string type: object type: object + idleConnectionTerminationPolicy: + default: Deferred + description: |- + idleConnectionTerminationPolicy maps directly to HAProxy's + idle-close-on-response option and controls whether HAProxy + keeps idle frontend connections open during a soft stop + (router reload). + + Allowed values for this field are "Immediate" and + "Deferred". The default value is "Deferred". + + When set to "Immediate", idle connections are closed + immediately during router reloads. This ensures immediate + propagation of route changes but may impact clients + sensitive to connection resets. + + When set to "Deferred", HAProxy will maintain idle + connections during a soft reload instead of closing them + immediately. These connections remain open until any of the + following occurs: + + - A new request is received on the connection, in which + case HAProxy handles it in the old process and closes + the connection after sending the response. + + - HAProxy's `timeout http-keep-alive` duration expires + (300 seconds in OpenShift's configuration, not + configurable). + + - The client's keep-alive timeout expires, causing the + client to close the connection. + + Setting Deferred can help prevent errors in clients or load + balancers that do not properly handle connection resets. + Additionally, this option allows you to retain the pre-2.4 + HAProxy behaviour: in HAProxy version 2.2 (OpenShift + versions < 4.14), maintaining idle connections during a + soft reload was the default behaviour, but starting with + HAProxy 2.4, the default changed to closing idle + connections immediately. + + Important Consideration: + + - Using Deferred will result in temporary inconsistencies + for the first request on each persistent connection + after a route update and router reload. This request + will be processed by the old HAProxy process using its + old configuration. Subsequent requests will use the + updated configuration. + + Operational Considerations: + + - Keeping idle connections open during reloads may lead + to an accumulation of old HAProxy processes if + connections remain idle for extended periods, + especially in environments where frequent reloads + occur. + + - Consider monitoring the number of HAProxy processes in + the router pods when Deferred is set. + + - You may need to enable or adjust the + `ingress.operator.openshift.io/hard-stop-after` + duration (configured via an annotation on the + IngressController resource) in environments with + frequent reloads to prevent resource exhaustion. + enum: + - Immediate + - Deferred + type: string logging: description: |- logging defines parameters for what should be logged where. If this diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 3c00fe2f032..b9260338b84 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -1020,24 +1020,25 @@ func (IngressControllerSetHTTPHeader) SwaggerDoc() map[string]string { } var map_IngressControllerSpec = map[string]string{ - "": "IngressControllerSpec is the specification of the desired behavior of the IngressController.", - "domain": "domain is a DNS name serviced by the ingress controller and is used to configure multiple features:\n\n* For the LoadBalancerService endpoint publishing strategy, domain is\n used to configure DNS records. See endpointPublishingStrategy.\n\n* When using a generated default certificate, the certificate will be valid\n for domain and its subdomains. See defaultCertificate.\n\n* The value is published to individual Route statuses so that end-users\n know where to target external DNS records.\n\ndomain must be unique among all IngressControllers, and cannot be updated.\n\nIf empty, defaults to ingress.config.openshift.io/cluster .spec.domain.", - "httpErrorCodePages": "httpErrorCodePages specifies a configmap with custom error pages. The administrator must create this configmap in the openshift-config namespace. This configmap should have keys in the format \"error-page-.http\", where is an HTTP error code. For example, \"error-page-503.http\" defines an error page for HTTP 503 responses. Currently only error pages for 503 and 404 responses can be customized. Each value in the configmap should be the full response, including HTTP headers. Eg- https://raw.githubusercontent.com/openshift/router/fadab45747a9b30cc3f0a4b41ad2871f95827a93/images/router/haproxy/conf/error-page-503.http If this field is empty, the ingress controller uses the default error pages.", - "replicas": "replicas is the desired number of ingress controller replicas. If unset, the default depends on the value of the defaultPlacement field in the cluster config.openshift.io/v1/ingresses status.\n\nThe value of replicas is set based on the value of a chosen field in the Infrastructure CR. If defaultPlacement is set to ControlPlane, the chosen field will be controlPlaneTopology. If it is set to Workers the chosen field will be infrastructureTopology. Replicas will then be set to 1 or 2 based whether the chosen field's value is SingleReplica or HighlyAvailable, respectively.\n\nThese defaults are subject to change.", - "endpointPublishingStrategy": "endpointPublishingStrategy is used to publish the ingress controller endpoints to other networks, enable load balancer integrations, etc.\n\nIf unset, the default is based on infrastructure.config.openshift.io/cluster .status.platform:\n\n AWS: LoadBalancerService (with External scope)\n Azure: LoadBalancerService (with External scope)\n GCP: LoadBalancerService (with External scope)\n IBMCloud: LoadBalancerService (with External scope)\n AlibabaCloud: LoadBalancerService (with External scope)\n Libvirt: HostNetwork\n\nAny other platform types (including None) default to HostNetwork.\n\nendpointPublishingStrategy cannot be updated.", - "defaultCertificate": "defaultCertificate is a reference to a secret containing the default certificate served by the ingress controller. When Routes don't specify their own certificate, defaultCertificate is used.\n\nThe secret must contain the following keys and data:\n\n tls.crt: certificate file contents\n tls.key: key file contents\n\nIf unset, a wildcard certificate is automatically generated and used. The certificate is valid for the ingress controller domain (and subdomains) and the generated certificate's CA will be automatically integrated with the cluster's trust store.\n\nIf a wildcard certificate is used and shared by multiple HTTP/2 enabled routes (which implies ALPN) then clients (i.e., notably browsers) are at liberty to reuse open connections. This means a client can reuse a connection to another route and that is likely to fail. This behaviour is generally known as connection coalescing.\n\nThe in-use certificate (whether generated or user-specified) will be automatically integrated with OpenShift's built-in OAuth server.", - "namespaceSelector": "namespaceSelector is used to filter the set of namespaces serviced by the ingress controller. This is useful for implementing shards.\n\nIf unset, the default is no filtering.", - "routeSelector": "routeSelector is used to filter the set of Routes serviced by the ingress controller. This is useful for implementing shards.\n\nIf unset, the default is no filtering.", - "nodePlacement": "nodePlacement enables explicit control over the scheduling of the ingress controller.\n\nIf unset, defaults are used. See NodePlacement for more details.", - "tlsSecurityProfile": "tlsSecurityProfile specifies settings for TLS connections for ingresscontrollers.\n\nIf unset, the default is based on the apiservers.config.openshift.io/cluster resource.\n\nNote that when using the Old, Intermediate, and Modern profile types, the effective profile configuration is subject to change between releases. For example, given a specification to use the Intermediate profile deployed on release X.Y.Z, an upgrade to release X.Y.Z+1 may cause a new profile configuration to be applied to the ingress controller, resulting in a rollout.", - "clientTLS": "clientTLS specifies settings for requesting and verifying client certificates, which can be used to enable mutual TLS for edge-terminated and reencrypt routes.", - "routeAdmission": "routeAdmission defines a policy for handling new route claims (for example, to allow or deny claims across namespaces).\n\nIf empty, defaults will be applied. See specific routeAdmission fields for details about their defaults.", - "logging": "logging defines parameters for what should be logged where. If this field is empty, operational logs are enabled but access logs are disabled.", - "httpHeaders": "httpHeaders defines policy for HTTP headers.\n\nIf this field is empty, the default values are used.", - "httpEmptyRequestsPolicy": "httpEmptyRequestsPolicy describes how HTTP connections should be handled if the connection times out before a request is received. Allowed values for this field are \"Respond\" and \"Ignore\". If the field is set to \"Respond\", the ingress controller sends an HTTP 400 or 408 response, logs the connection (if access logging is enabled), and counts the connection in the appropriate metrics. If the field is set to \"Ignore\", the ingress controller closes the connection without sending a response, logging the connection, or incrementing metrics. The default value is \"Respond\".\n\nTypically, these connections come from load balancers' health probes or Web browsers' speculative connections (\"preconnect\") and can be safely ignored. However, these requests may also be caused by network errors, and so setting this field to \"Ignore\" may impede detection and diagnosis of problems. In addition, these requests may be caused by port scans, in which case logging empty requests may aid in detecting intrusion attempts.", - "tuningOptions": "tuningOptions defines parameters for adjusting the performance of ingress controller pods. All fields are optional and will use their respective defaults if not set. See specific tuningOptions fields for more details.\n\nSetting fields within tuningOptions is generally not recommended. The default values are suitable for most configurations.", - "unsupportedConfigOverrides": "unsupportedConfigOverrides allows specifying unsupported configuration options. Its use is unsupported.", - "httpCompression": "httpCompression defines a policy for HTTP traffic compression. By default, there is no HTTP compression.", + "": "IngressControllerSpec is the specification of the desired behavior of the IngressController.", + "domain": "domain is a DNS name serviced by the ingress controller and is used to configure multiple features:\n\n* For the LoadBalancerService endpoint publishing strategy, domain is\n used to configure DNS records. See endpointPublishingStrategy.\n\n* When using a generated default certificate, the certificate will be valid\n for domain and its subdomains. See defaultCertificate.\n\n* The value is published to individual Route statuses so that end-users\n know where to target external DNS records.\n\ndomain must be unique among all IngressControllers, and cannot be updated.\n\nIf empty, defaults to ingress.config.openshift.io/cluster .spec.domain.", + "httpErrorCodePages": "httpErrorCodePages specifies a configmap with custom error pages. The administrator must create this configmap in the openshift-config namespace. This configmap should have keys in the format \"error-page-.http\", where is an HTTP error code. For example, \"error-page-503.http\" defines an error page for HTTP 503 responses. Currently only error pages for 503 and 404 responses can be customized. Each value in the configmap should be the full response, including HTTP headers. Eg- https://raw.githubusercontent.com/openshift/router/fadab45747a9b30cc3f0a4b41ad2871f95827a93/images/router/haproxy/conf/error-page-503.http If this field is empty, the ingress controller uses the default error pages.", + "replicas": "replicas is the desired number of ingress controller replicas. If unset, the default depends on the value of the defaultPlacement field in the cluster config.openshift.io/v1/ingresses status.\n\nThe value of replicas is set based on the value of a chosen field in the Infrastructure CR. If defaultPlacement is set to ControlPlane, the chosen field will be controlPlaneTopology. If it is set to Workers the chosen field will be infrastructureTopology. Replicas will then be set to 1 or 2 based whether the chosen field's value is SingleReplica or HighlyAvailable, respectively.\n\nThese defaults are subject to change.", + "endpointPublishingStrategy": "endpointPublishingStrategy is used to publish the ingress controller endpoints to other networks, enable load balancer integrations, etc.\n\nIf unset, the default is based on infrastructure.config.openshift.io/cluster .status.platform:\n\n AWS: LoadBalancerService (with External scope)\n Azure: LoadBalancerService (with External scope)\n GCP: LoadBalancerService (with External scope)\n IBMCloud: LoadBalancerService (with External scope)\n AlibabaCloud: LoadBalancerService (with External scope)\n Libvirt: HostNetwork\n\nAny other platform types (including None) default to HostNetwork.\n\nendpointPublishingStrategy cannot be updated.", + "defaultCertificate": "defaultCertificate is a reference to a secret containing the default certificate served by the ingress controller. When Routes don't specify their own certificate, defaultCertificate is used.\n\nThe secret must contain the following keys and data:\n\n tls.crt: certificate file contents\n tls.key: key file contents\n\nIf unset, a wildcard certificate is automatically generated and used. The certificate is valid for the ingress controller domain (and subdomains) and the generated certificate's CA will be automatically integrated with the cluster's trust store.\n\nIf a wildcard certificate is used and shared by multiple HTTP/2 enabled routes (which implies ALPN) then clients (i.e., notably browsers) are at liberty to reuse open connections. This means a client can reuse a connection to another route and that is likely to fail. This behaviour is generally known as connection coalescing.\n\nThe in-use certificate (whether generated or user-specified) will be automatically integrated with OpenShift's built-in OAuth server.", + "namespaceSelector": "namespaceSelector is used to filter the set of namespaces serviced by the ingress controller. This is useful for implementing shards.\n\nIf unset, the default is no filtering.", + "routeSelector": "routeSelector is used to filter the set of Routes serviced by the ingress controller. This is useful for implementing shards.\n\nIf unset, the default is no filtering.", + "nodePlacement": "nodePlacement enables explicit control over the scheduling of the ingress controller.\n\nIf unset, defaults are used. See NodePlacement for more details.", + "tlsSecurityProfile": "tlsSecurityProfile specifies settings for TLS connections for ingresscontrollers.\n\nIf unset, the default is based on the apiservers.config.openshift.io/cluster resource.\n\nNote that when using the Old, Intermediate, and Modern profile types, the effective profile configuration is subject to change between releases. For example, given a specification to use the Intermediate profile deployed on release X.Y.Z, an upgrade to release X.Y.Z+1 may cause a new profile configuration to be applied to the ingress controller, resulting in a rollout.", + "clientTLS": "clientTLS specifies settings for requesting and verifying client certificates, which can be used to enable mutual TLS for edge-terminated and reencrypt routes.", + "routeAdmission": "routeAdmission defines a policy for handling new route claims (for example, to allow or deny claims across namespaces).\n\nIf empty, defaults will be applied. See specific routeAdmission fields for details about their defaults.", + "logging": "logging defines parameters for what should be logged where. If this field is empty, operational logs are enabled but access logs are disabled.", + "httpHeaders": "httpHeaders defines policy for HTTP headers.\n\nIf this field is empty, the default values are used.", + "httpEmptyRequestsPolicy": "httpEmptyRequestsPolicy describes how HTTP connections should be handled if the connection times out before a request is received. Allowed values for this field are \"Respond\" and \"Ignore\". If the field is set to \"Respond\", the ingress controller sends an HTTP 400 or 408 response, logs the connection (if access logging is enabled), and counts the connection in the appropriate metrics. If the field is set to \"Ignore\", the ingress controller closes the connection without sending a response, logging the connection, or incrementing metrics. The default value is \"Respond\".\n\nTypically, these connections come from load balancers' health probes or Web browsers' speculative connections (\"preconnect\") and can be safely ignored. However, these requests may also be caused by network errors, and so setting this field to \"Ignore\" may impede detection and diagnosis of problems. In addition, these requests may be caused by port scans, in which case logging empty requests may aid in detecting intrusion attempts.", + "tuningOptions": "tuningOptions defines parameters for adjusting the performance of ingress controller pods. All fields are optional and will use their respective defaults if not set. See specific tuningOptions fields for more details.\n\nSetting fields within tuningOptions is generally not recommended. The default values are suitable for most configurations.", + "unsupportedConfigOverrides": "unsupportedConfigOverrides allows specifying unsupported configuration options. Its use is unsupported.", + "httpCompression": "httpCompression defines a policy for HTTP traffic compression. By default, there is no HTTP compression.", + "idleConnectionTerminationPolicy": "idleConnectionTerminationPolicy maps directly to HAProxy's idle-close-on-response option and controls whether HAProxy keeps idle frontend connections open during a soft stop (router reload).\n\nAllowed values for this field are \"Immediate\" and \"Deferred\". The default value is \"Deferred\".\n\nWhen set to \"Immediate\", idle connections are closed immediately during router reloads. This ensures immediate propagation of route changes but may impact clients sensitive to connection resets.\n\nWhen set to \"Deferred\", HAProxy will maintain idle connections during a soft reload instead of closing them immediately. These connections remain open until any of the following occurs:\n\n - A new request is received on the connection, in which\n case HAProxy handles it in the old process and closes\n the connection after sending the response.\n\n - HAProxy's `timeout http-keep-alive` duration expires\n (300 seconds in OpenShift's configuration, not\n configurable).\n\n - The client's keep-alive timeout expires, causing the\n client to close the connection.\n\nSetting Deferred can help prevent errors in clients or load balancers that do not properly handle connection resets. Additionally, this option allows you to retain the pre-2.4 HAProxy behaviour: in HAProxy version 2.2 (OpenShift versions < 4.14), maintaining idle connections during a soft reload was the default behaviour, but starting with HAProxy 2.4, the default changed to closing idle connections immediately.\n\nImportant Consideration:\n\n - Using Deferred will result in temporary inconsistencies\n for the first request on each persistent connection\n after a route update and router reload. This request\n will be processed by the old HAProxy process using its\n old configuration. Subsequent requests will use the\n updated configuration.\n\nOperational Considerations:\n\n - Keeping idle connections open during reloads may lead\n to an accumulation of old HAProxy processes if\n connections remain idle for extended periods,\n especially in environments where frequent reloads\n occur.\n\n - Consider monitoring the number of HAProxy processes in\n the router pods when Deferred is set.\n\n - You may need to enable or adjust the\n `ingress.operator.openshift.io/hard-stop-after`\n duration (configured via an annotation on the\n IngressController resource) in environments with\n frequent reloads to prevent resource exhaustion.", } func (IngressControllerSpec) SwaggerDoc() map[string]string {