Support NodePortLocal on Antrea Windows Agent and Update NPL annotati…

…on (#3453)

* Support NodePortLocal on Antrea Windows Agent

  * Support NodePortLocal rules on by using NetNatStaticMapping on windows
  * Support NPL agent on Windows platform
  * Require the same Antrea NPL configuration as Linux

* Support windows agent configmap for mutateAntreaConfigMap

  Currently mutateAntreaConfigMap can only support linux agent configmap.
  This patch will support windows configmap processing.

* Support creating nginx pods on windows

  * Add related image to support more e2e tests on windows.

* Support NPL e2e test for windows nodes

  * Add NPL Netnat rules check on windows
  * Replace busybox with agnhost as the client pod
  * Still skip windows test by default due to the ovs HNSCall issue

* Support protocol-independent NPL annotation

  * Add protocol string to new NPLAnnotation set the protocols map as a deprecated value.
  * Support Node port for UDP and TCP using the different number for a single Pod on Windows.

* Update docs for NodePortLocal

  * Update annotation example with new protocol field.
  * Update NPL windows support feature.

Closes #3826

Signed-off-by: Shuyang Xin <gavinx@vmware.com>

build/yamls/antrea-windows.yml

@@ -28,6 +28,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePortLocal feature to make the Pods reachable externally through NodePort
+    #  NodePortLocal: true
+
     # Enable flowexporter which exports polled conntrack connections as IPFIX flow records from each agent to a configured collector.
     #  FlowExporter: false
 
@@ -131,6 +134,17 @@ data:
       # Note that this option is experimental. If kube-proxy is removed, option kubeAPIServerOverride must be used to access
       # apiserver directly.
       #proxyAll: false
+
+    nodePortLocal:
+    # Enable NodePortLocal, a feature used to make Pods reachable using port forwarding on the host. To
+    # enable this feature, you need to set "enable" to true, and ensure that the NodePortLocal feature
+    # gate is also enabled (which is the default).
+    #  enable: false
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port
+    # from that range will be assigned whenever a Pod's container defines a specific port to be exposed
+    # (each container can define a list of ports as pod.spec.containers[].ports), and all Node traffic
+    # directed to that port will be forwarded to the Pod.
+    #  portRange: 61000-62000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -149,7 +163,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-windows-config-mf82kffb45
+  name: antrea-windows-config-8kfkb8t957
   namespace: kube-system
 ---
 apiVersion: apps/v1
@@ -237,7 +251,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-windows-config-mf82kffb45
+          name: antrea-windows-config-8kfkb8t957
         name: antrea-windows-config
       - configMap:
           defaultMode: 420

build/yamls/windows/base/conf/antrea-agent.conf

@@ -10,6 +10,9 @@ featureGates:
 # this flag will not take effect.
 #  EndpointSlice: false
 
+# Enable NodePortLocal feature to make the Pods reachable externally through NodePort
+#  NodePortLocal: true
+
 # Enable flowexporter which exports polled conntrack connections as IPFIX flow records from each agent to a configured collector.
 #  FlowExporter: false
 
@@ -113,3 +116,14 @@ antreaProxy:
   # Note that this option is experimental. If kube-proxy is removed, option kubeAPIServerOverride must be used to access
   # apiserver directly.
   #proxyAll: false
+
+nodePortLocal:
+# Enable NodePortLocal, a feature used to make Pods reachable using port forwarding on the host. To
+# enable this feature, you need to set "enable" to true, and ensure that the NodePortLocal feature
+# gate is also enabled (which is the default).
+#  enable: false
+# Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port
+# from that range will be assigned whenever a Pod's container defines a specific port to be exposed
+# (each container can define a list of ports as pod.spec.containers[].ports), and all Node traffic
+# directed to that port will be forwarded to the Pod.
+#  portRange: 61000-62000

docs/node-port-local.md

@@ -6,6 +6,7 @@
 - [What is NodePortLocal?](#what-is-nodeportlocal)
 - [Prerequisites](#prerequisites)
 - [Usage](#usage)
+  - [Usage pre Antrea v1.7](#usage-pre-antrea-v17)
   - [Usage pre Antrea v1.4](#usage-pre-antrea-v14)
   - [Usage pre Antrea v1.2](#usage-pre-antrea-v12)
 - [Limitations](#limitations)
@@ -29,7 +30,7 @@ directly to backend Pods.
 NodePortLocal was introduced in v0.13 as an alpha feature, and was graduated to
 beta in v1.4, at which time it was enabled by default. Prior to v1.4, a feature
 gate, `NodePortLocal`, must be enabled on the antrea-agent for the feature to
-work.
+work. Starting from Antrea v1.7, NPL is supported on the Windows antrea-agent.
 
 ## Usage
 
@@ -114,12 +115,12 @@ metadata:
   labels:
     app: nginx
   annotations:
-    nodeportlocal.antrea.io: '[{"podPort":8080,"nodeIP":"10.10.10.10","nodePort":61002}]'
+    nodeportlocal.antrea.io: '[{"podPort":8080,"nodeIP":"10.10.10.10","nodePort":61002,"protocol":"tcp","protocols":["tcp"]}]'
 ...
 ```
 
 This annotation indicates that port 8080 of the Pod can be reached through port
-61002 of the Node with IP Address 10.10.10.10.
+61002 of the Node with IP Address 10.10.10.10 for TCP traffic.
 
 The `nodeportlocal.antrea.io` annotation is generated and managed by Antrea. It
 is not meant to be created or modified by users directly. A user-provided
@@ -131,6 +132,38 @@ NodePortLocal can only be used with Services of type `ClusterIP` or
 Services of type `NodePort` or `ExternalName`. The annotation also has no effect
 for Services with an empty or missing Selector.
 
+Starting from the Antrea v1.7 minor release, the `protocols` field in the
+annotation is deprecated. The array contains a single member, equal to the
+`protocol` field.
+The `protocols` field will be removed from Antrea for minor releases post March 2023,
+as per our deprecation policy.
+
+### Usage pre Antrea v1.7
+
+Prior to the Antrea v1.7 minor release, the `nodeportlocal.antrea.io` annotation
+could contain multiple members in `protocols`.
+An example may look like this:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-6799fc88d8-9rx8z
+  labels:
+    app: nginx
+  annotations:
+    nodeportlocal.antrea.io: '[{"podPort":8080,"nodeIP":"10.10.10.10","nodePort":61002}, "protocols":["tcp","udp"]]'
+...
+```
+
+This annotation indicates that port 8080 of the Pod can be reached through port
+61002 of the Node with IP Address 10.10.10.10 for both TCP and UDP traffic.
+
+Prior to v1.7, the implementation would always allocate the same nodePort value
+for all the protocols exposed for a given podPort.
+Starting with v1.7, there will be multiple annotations for the different protocols
+for a given podPort, and the allocated nodePort may be different for each one.
+
 ### Usage pre Antrea v1.4
 
 Prior to the Antrea v1.4 minor release, the `nodePortLocal` option group in the
@@ -179,8 +212,8 @@ mapped.
 
 ## Limitations
 
-This feature is currently only supported for Nodes running Linux with IPv4
-addresses. Only TCP & UDP Service ports are supported (not SCTP).
+This feature is currently only supported for Nodes running Linux or Windows
+with IPv4 addresses. Only TCP & UDP Service ports are supported (not SCTP).
 
 ## Integrations with External Load Balancers
 

pkg/agent/nodeportlocal/k8s/annotations.go

@@ -1,6 +1,3 @@
-//go:build !windows
-// +build !windows
-
 // Copyright 2020 Antrea Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -42,7 +39,8 @@ type NPLAnnotation struct {
 	PodPort   int      `json:"podPort"`
 	NodeIP    string   `json:"nodeIP"`
 	NodePort  int      `json:"nodePort"`
-	Protocols []string `json:"protocols"`
+	Protocol  string   `json:"protocol"`
+	Protocols []string `json:"protocols"` // deprecated, array with a single member which is equal to the Protocol field
 }
 
 func toJSON(serialize interface{}) string {

pkg/agent/nodeportlocal/k8s/npl_controller.go

@@ -1,6 +1,3 @@
-//go:build !windows
-// +build !windows
-
 // Copyright 2020 Antrea Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -450,7 +447,7 @@ func (c *NPLController) handleAddUpdatePod(key string, obj interface{}) error {
 		}
 	}
 
-	nplAnnotationsRequiredMap := map[int]NPLAnnotation{}
+	nplAnnotationsRequiredMap := map[string]NPLAnnotation{}
 	nplAnnotationsRequired := []NPLAnnotation{}
 
 	hostPorts := make(map[string]int)
@@ -494,7 +491,7 @@ func (c *NPLController) handleAddUpdatePod(key string, obj interface{}) error {
 			return fmt.Errorf("failed to parse port number and protocol from %s for Pod %s: %v", targetPortProto, key, err)
 		}
 		podPorts[targetPortProto] = struct{}{}
-		portData := c.portTable.GetEntry(podIP, port)
+		portData := c.portTable.GetEntry(podIP, port, protocol)
 		if portData != nil && !portData.ProtocolInUse(protocol) {
 			// If the PortTable has an entry for the Pod but does not have an
 			// entry with protocol, we enforce AddRule for the missing Protocol.
@@ -513,14 +510,12 @@ func (c *NPLController) handleAddUpdatePod(key string, obj interface{}) error {
 			nodePort = portData.NodePort
 		}
 
-		if val, ok := nplAnnotationsRequiredMap[nodePort]; ok {
-			val.Protocols = append(val.Protocols, protocol)
-			nplAnnotationsRequiredMap[nodePort] = val
-		} else {
-			nplAnnotationsRequiredMap[nodePort] = NPLAnnotation{
+		if _, ok := nplAnnotationsRequiredMap[portcache.NodePortProtoFormat(nodePort, protocol)]; !ok {
+			nplAnnotationsRequiredMap[portcache.NodePortProtoFormat(nodePort, protocol)] = NPLAnnotation{
 				PodPort:   port,
 				NodeIP:    pod.Status.HostIP,
 				NodePort:  nodePort,
+				Protocol:  protocol,
 				Protocols: []string{protocol},
 			}
 		}
@@ -607,6 +602,7 @@ func (c *NPLController) waitForRulesInitialization() {
 				NodePort:  npl.NodePort,
 				PodPort:   npl.PodPort,
 				PodIP:     pod.Status.PodIP,
+				Protocol:  npl.Protocol,
 				Protocols: npl.Protocols,
 			})
 		}

pkg/agent/nodeportlocal/npl_agent_init.go

@@ -1,6 +1,3 @@
-//go:build !windows
-// +build !windows
-
 // Copyright 2020 Antrea Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");

pkg/agent/nodeportlocal/npl_agent_test.go

@@ -66,7 +66,7 @@ const (
 
 func newPortTable(mockIPTables rules.PodPortRules, mockPortOpener portcache.LocalPortOpener) *portcache.PortTable {
 	return &portcache.PortTable{
-		NodePortTable:    make(map[int]*portcache.NodePortData),
+		NodePortTable:    make(map[string]*portcache.NodePortData),
 		PodEndpointTable: make(map[string]*portcache.NodePortData),
 		StartPort:        defaultStartPort,
 		EndPort:          defaultEndPort,