Skip to content

Commit 81607b6

Browse files
committed
[Windows] CNI Server installs OpenFlow entries after PortStatus message is received
Signed-off-by: Wenying Dong <wenyingd@vmware.com>
1 parent b4916eb commit 81607b6

25 files changed

+942
-136
lines changed

cmd/antrea-agent/agent.go

+1
Original file line numberDiff line numberDiff line change
@@ -600,6 +600,7 @@ func run(o *Options) error {
600600
o.config.CNISocket,
601601
o.config.HostProcPathPrefix,
602602
nodeConfig,
603+
localPodInformer.Get(),
603604
k8sClient,
604605
routeClient,
605606
isChaining,

go.mod

+4-5
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ module antrea.io/antrea
33
go 1.23.0
44

55
require (
6-
antrea.io/libOpenflow v0.14.0
7-
antrea.io/ofnet v0.12.0
6+
antrea.io/libOpenflow v0.15.0
7+
antrea.io/ofnet v0.14.0
88
github.com/ClickHouse/clickhouse-go/v2 v2.6.1
99
github.com/DATA-DOG/go-sqlmock v1.5.2
1010
github.com/Mellanox/sriovnet v1.1.0
@@ -113,10 +113,9 @@ require (
113113
github.com/aws/smithy-go v1.12.1 // indirect
114114
github.com/beorn7/perks v1.0.1 // indirect
115115
github.com/blang/semver/v4 v4.0.0 // indirect
116-
github.com/cenk/hub v1.0.1 // indirect
117116
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
118-
github.com/cenkalti/hub v1.0.1 // indirect
119-
github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa // indirect
117+
github.com/cenkalti/hub v1.0.2 // indirect
118+
github.com/cenkalti/rpc2 v1.0.3 // indirect
120119
github.com/cespare/xxhash/v2 v2.3.0 // indirect
121120
github.com/chai2010/gettext-go v1.0.2 // indirect
122121
github.com/containerd/cgroups v1.1.0 // indirect

go.sum

+8-10
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
antrea.io/libOpenflow v0.14.0 h1:6MS1E52nGQyz/AJ8j3OrotgAc/1ubef5vc5i8ytIxFE=
2-
antrea.io/libOpenflow v0.14.0/go.mod h1:U8YR0ithWrjwLdUUhyeCsYnlKg/mEFjG5CbPNt1V+j0=
3-
antrea.io/ofnet v0.12.0 h1:pgygAsEZJUPK/kGmeuIesDh5hoGLpYeavSLdG/Dp8Ao=
4-
antrea.io/ofnet v0.12.0/go.mod h1:MB3qaInEimj+T8qtpBcIQK+EqeNiY1S/WbUdGk+TzNg=
1+
antrea.io/libOpenflow v0.15.0 h1:wGk+IVCf8piGZgC4+lbf4qfGrJG5ikzfq5Y1T5LzqmI=
2+
antrea.io/libOpenflow v0.15.0/go.mod h1:Mq1JEjYrb6eTVA7qjZRj9plVTKcsLM8wnQ87sLLYuiQ=
3+
antrea.io/ofnet v0.14.0 h1:BGOqg5DdRkvxpBqyoEgWmvGd4EvpacdU/Py1s6qOvSc=
4+
antrea.io/ofnet v0.14.0/go.mod h1:W5JPYFFcRM7tLwsItgmsKqIhtW/QofyIeNsUIecFaBo=
55
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
66
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
77
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -114,14 +114,12 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn
114114
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
115115
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
116116
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
117-
github.com/cenk/hub v1.0.1 h1:RBwXNOF4a8KjD8BJ08XqN8KbrqaGiQLDrgvUGJSHuPA=
118-
github.com/cenk/hub v1.0.1/go.mod h1:rJM1LNAW0ppT8FMMuPK6c2NP/R2nH/UthtuRySSaf6Y=
119117
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
120118
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
121-
github.com/cenkalti/hub v1.0.1 h1:UMtjc6dHSaOQTO15SVA50MBIR9zQwvsukQupDrkIRtg=
122-
github.com/cenkalti/hub v1.0.1/go.mod h1:tcYwtS3a2d9NO/0xDXVJWx3IedurUjYCqFCmpi0lpHs=
123-
github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa h1:t+iWhuJE2aropY4uxKMVbyP+IJ29o422f7YAd73aTjg=
124-
github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa/go.mod h1:v2npkhrXyk5BCnkNIiPdRI23Uq6uWPUQGL2hnRcRr/M=
119+
github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8=
120+
github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU=
121+
github.com/cenkalti/rpc2 v1.0.3 h1:OkMsNP/sP9seN1VRCLqhX1xkVGHPoLwWS6fZR14Ji/k=
122+
github.com/cenkalti/rpc2 v1.0.3/go.mod h1:2yfU5b86vOr16+iY1jN3MvT6Kxc9Nf8j5iZWwUf7iaw=
125123
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
126124
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
127125
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

pkg/agent/cniserver/pod_configuration.go

+19-7
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import (
2525
"github.com/containernetworking/cni/pkg/version"
2626
corev1 "k8s.io/api/core/v1"
2727
"k8s.io/apimachinery/pkg/util/sets"
28+
clientset "k8s.io/client-go/kubernetes"
29+
v1 "k8s.io/client-go/listers/core/v1"
2830
"k8s.io/klog/v2"
2931

3032
"antrea.io/antrea/pkg/agent/cniserver/ipam"
@@ -76,9 +78,11 @@ type podConfigurator struct {
7678
// isSecondaryNetwork is true if this instance of podConfigurator is used to configure
7779
// Pod secondary network interfaces.
7880
isSecondaryNetwork bool
81+
podIfMonitor *podIfaceMonitor
7982
}
8083

8184
func newPodConfigurator(
85+
kubeClient clientset.Interface,
8286
ovsBridgeClient ovsconfig.OVSBridgeClient,
8387
ofClient openflow.Client,
8488
routeClient route.Interface,
@@ -88,11 +92,13 @@ func newPodConfigurator(
8892
isOvsHardwareOffloadEnabled bool,
8993
disableTXChecksumOffload bool,
9094
podUpdateNotifier channel.Notifier,
95+
podLister v1.PodLister,
9196
) (*podConfigurator, error) {
9297
ifConfigurator, err := newInterfaceConfigurator(ovsDatapathType, isOvsHardwareOffloadEnabled, disableTXChecksumOffload)
9398
if err != nil {
9499
return nil, err
95100
}
101+
ifMonitor := newPodInterfaceMonitor(kubeClient, podLister, ofClient, ifaceStore, podUpdateNotifier)
96102
return &podConfigurator{
97103
ovsBridgeClient: ovsBridgeClient,
98104
ofClient: ofClient,
@@ -101,6 +107,7 @@ func newPodConfigurator(
101107
gatewayMAC: gatewayMAC,
102108
ifConfigurator: ifConfigurator,
103109
podUpdateNotifier: podUpdateNotifier,
110+
podIfMonitor: ifMonitor,
104111
}, nil
105112
}
106113

@@ -166,13 +173,13 @@ func getContainerIPsString(ips []net.IP) string {
166173
// not created for a Pod interface.
167174
func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *interfacestore.OVSPortConfig) *interfacestore.InterfaceConfig {
168175
if portData.ExternalIDs == nil {
169-
klog.V(2).Infof("OVS port %s has no external_ids", portData.Name)
176+
klog.V(2).InfoS("OVS port has no external_ids", "port", portData.Name)
170177
return nil
171178
}
172179

173180
containerID, found := portData.ExternalIDs[ovsExternalIDContainerID]
174181
if !found {
175-
klog.V(2).Infof("OVS port %s has no %s in external_ids", portData.Name, ovsExternalIDContainerID)
182+
klog.V(2).InfoS("OVS port has no containerID in external_ids", "port", portData.Name)
176183
return nil
177184
}
178185

@@ -187,8 +194,7 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in
187194

188195
containerMAC, err := net.ParseMAC(portData.ExternalIDs[ovsExternalIDMAC])
189196
if err != nil {
190-
klog.Errorf("Failed to parse MAC address from OVS external config %s: %v",
191-
portData.ExternalIDs[ovsExternalIDMAC], err)
197+
klog.ErrorS(err, "Failed to parse MAC address from OVS external config")
192198
}
193199
podName, _ := portData.ExternalIDs[ovsExternalIDPodName]
194200
podNamespace, _ := portData.ExternalIDs[ovsExternalIDPodNamespace]
@@ -279,7 +285,7 @@ func (pc *podConfigurator) createOVSPort(ovsPortName string, ovsAttachInfo map[s
279285
func (pc *podConfigurator) removeInterfaces(containerID string) error {
280286
containerConfig, found := pc.ifaceStore.GetContainerInterface(containerID)
281287
if !found {
282-
klog.V(2).Infof("Did not find the port for container %s in local cache", containerID)
288+
klog.V(2).InfoS("Did not find the port for container in local cache", "container", containerID)
283289
return nil
284290
}
285291

@@ -498,7 +504,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain
498504
// disconnectInterfaceFromOVS disconnects an existing interface from ovs br-int.
499505
func (pc *podConfigurator) disconnectInterfaceFromOVS(containerConfig *interfacestore.InterfaceConfig) error {
500506
containerID := containerConfig.ContainerID
501-
klog.V(2).Infof("Deleting Openflow entries for container %s", containerID)
507+
klog.V(2).InfoS("Deleting Openflow entries for container", "container", containerID)
502508
if !pc.isSecondaryNetwork {
503509
if err := pc.ofClient.UninstallPodFlows(containerConfig.InterfaceName); err != nil {
504510
return fmt.Errorf("failed to delete Openflow entries for container %s: %v", containerID, err)
@@ -513,6 +519,12 @@ func (pc *podConfigurator) disconnectInterfaceFromOVS(containerConfig *interface
513519
if err := pc.ovsBridgeClient.DeletePort(containerConfig.PortUUID); err != nil {
514520
return fmt.Errorf("failed to delete OVS port for container %s interface %s: %v", containerID, containerConfig.InterfaceName, err)
515521
}
522+
523+
// Remove unready Pod info from local cache when Pod is deleted. This is called only on Windows.
524+
if pc.podIfMonitor != nil {
525+
pc.podIfMonitor.deleteUnreadyPod(containerConfig.InterfaceName)
526+
}
527+
516528
// Remove container configuration from cache.
517529
pc.ifaceStore.DeleteInterface(containerConfig)
518530
if !pc.isSecondaryNetwork {
@@ -558,7 +570,7 @@ func (pc *podConfigurator) connectInterceptedInterface(
558570
func (pc *podConfigurator) disconnectInterceptedInterface(podName, podNamespace, containerID string) error {
559571
containerConfig, found := pc.ifaceStore.GetContainerInterface(containerID)
560572
if !found {
561-
klog.V(2).Infof("Did not find the port for container %s in local cache", containerID)
573+
klog.V(2).InfoS("Did not find the port for container in local cache", "container", containerID)
562574
return nil
563575
}
564576
for _, ip := range containerConfig.IPs {

pkg/agent/cniserver/pod_configuration_linux_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/stretchr/testify/assert"
2525
"github.com/stretchr/testify/require"
2626
"go.uber.org/mock/gomock"
27+
fakeclientset "k8s.io/client-go/kubernetes/fake"
2728

2829
"antrea.io/antrea/pkg/agent/cniserver/ipam"
2930
ipamtest "antrea.io/antrea/pkg/agent/cniserver/ipam/testing"
@@ -682,12 +683,13 @@ func TestDeleteVLANSecondaryInterface(t *testing.T) {
682683
}
683684

684685
func createPodConfigurator(controller *gomock.Controller, testIfaceConfigurator *fakeInterfaceConfigurator) *podConfigurator {
686+
kubeClient := fakeclientset.NewSimpleClientset()
685687
gwMAC, _ := net.ParseMAC("00:00:11:11:11:11")
686688
mockOVSBridgeClient = ovsconfigtest.NewMockOVSBridgeClient(controller)
687689
mockOFClient = openflowtest.NewMockClient(controller)
688690
ifaceStore = interfacestore.NewInterfaceStore()
689691
mockRoute = routetest.NewMockInterface(controller)
690-
configurator, _ := newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100))
692+
configurator, _ := newPodConfigurator(kubeClient, mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
691693
configurator.ifConfigurator = testIfaceConfigurator
692694
return configurator
693695
}

pkg/agent/cniserver/pod_configuration_windows.go

+9-28
Original file line numberDiff line numberDiff line change
@@ -18,45 +18,27 @@
1818
package cniserver
1919

2020
import (
21-
"fmt"
22-
23-
current "github.com/containernetworking/cni/pkg/types/100"
2421
"k8s.io/klog/v2"
2522

2623
"antrea.io/antrea/pkg/agent/cniserver/ipam"
2724
"antrea.io/antrea/pkg/agent/interfacestore"
28-
"antrea.io/antrea/pkg/agent/types"
29-
"antrea.io/antrea/pkg/util/k8s"
25+
current "github.com/containernetworking/cni/pkg/types/100"
3026
)
3127

3228
// connectInterfaceToOVSAsync waits for an interface to be created and connects it to OVS br-int asynchronously
3329
// in another goroutine. The function is for containerd runtime. The host interface is created after
3430
// CNI call completes.
3531
func (pc *podConfigurator) connectInterfaceToOVSAsync(ifConfig *interfacestore.InterfaceConfig, containerAccess *containerAccessArbitrator) error {
3632
ovsPortName := ifConfig.InterfaceName
33+
// Add the OVS port into unReadyPorts. This operation is performed before we update OVSDB, otherwise we
34+
// need to think about the race condition between the current goroutine with the listener.
35+
// Note, we may add OVS port into "unReadyOVSPorts" map even if the update OVSDB operation is failed,
36+
// because it is also a case that the Pod's networking is not ready.
37+
pc.podIfMonitor.addUnreadyPodInterface(ifConfig)
3738
return pc.ifConfigurator.addPostInterfaceCreateHook(ifConfig.ContainerID, ovsPortName, containerAccess, func() error {
3839
if err := pc.ovsBridgeClient.SetInterfaceType(ovsPortName, "internal"); err != nil {
3940
return err
4041
}
41-
ofPort, err := pc.ovsBridgeClient.GetOFPort(ovsPortName, true)
42-
if err != nil {
43-
return err
44-
}
45-
containerID := ifConfig.ContainerID
46-
klog.V(2).Infof("Setting up Openflow entries for container %s", containerID)
47-
if err := pc.ofClient.InstallPodFlows(ovsPortName, ifConfig.IPs, ifConfig.MAC, uint32(ofPort), ifConfig.VLANID, nil); err != nil {
48-
return fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err)
49-
}
50-
// Update interface config with the ofPort.
51-
ifConfig.OVSPortConfig.OFPort = ofPort
52-
// Notify the Pod update event to required components.
53-
event := types.PodUpdate{
54-
PodName: ifConfig.PodName,
55-
PodNamespace: ifConfig.PodNamespace,
56-
IsAdd: true,
57-
ContainerID: ifConfig.ContainerID,
58-
}
59-
pc.podUpdateNotifier.Notify(event)
6042
return nil
6143
})
6244
}
@@ -75,7 +57,7 @@ func (pc *podConfigurator) connectInterfaceToOVS(
7557
// Because of this, we need to wait asynchronously for the interface to be created: we create the OVS port
7658
// and set the OVS Interface type "" first, and change the OVS Interface type to "internal" to connect to the
7759
// container interface after it is created. After OVS connects to the container interface, an OFPort is allocated.
78-
klog.V(2).Infof("Adding OVS port %s for container %s", ovsPortName, containerID)
60+
klog.V(2).InfoS("Adding OVS port for container", "port", ovsPortName, "container", containerID)
7961
ovsAttachInfo := BuildOVSPortExternalIDs(containerConfig)
8062
portUUID, err := pc.createOVSPort(ovsPortName, ovsAttachInfo, containerConfig.VLANID)
8163
if err != nil {
@@ -105,7 +87,7 @@ func (pc *podConfigurator) configureInterfaces(
10587
// See: https://github.com/kubernetes/kubernetes/issues/57253#issuecomment-358897721.
10688
interfaceConfig, found := pc.ifaceStore.GetContainerInterface(containerID)
10789
if found {
108-
klog.V(2).Infof("Found an existing OVS port for container %s, returning", containerID)
90+
klog.V(2).InfoS("Found an existing OVS port for container, returning", "container", containerID)
10991
mac := interfaceConfig.MAC.String()
11092
hostIface := &current.Interface{
11193
Name: interfaceConfig.InterfaceName,
@@ -128,9 +110,8 @@ func (pc *podConfigurator) configureInterfaces(
128110
func (pc *podConfigurator) reconcileMissingPods(ifConfigs []*interfacestore.InterfaceConfig, containerAccess *containerAccessArbitrator) {
129111
for i := range ifConfigs {
130112
ifaceConfig := ifConfigs[i]
131-
pod := k8s.NamespacedName(ifaceConfig.PodNamespace, ifaceConfig.PodName)
132113
if err := pc.connectInterfaceToOVSAsync(ifaceConfig, containerAccess); err != nil {
133-
klog.Errorf("Failed to reconcile Pod %s: %v", pod, err)
114+
klog.ErrorS(err, "Failed to reconcile Pod", "name", ifaceConfig.PodName, "Namespace", ifaceConfig.PodNamespace)
134115
}
135116
}
136117
}

0 commit comments

Comments
 (0)