diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index a0bd6756d7..11a4cbf25a 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -465,7 +465,6 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo - HostSecondaryIPInfo HostIPInfo // NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC NICType NICType InterfaceName string @@ -475,15 +474,12 @@ type PodIpInfo struct { SkipDefaultRoutes bool // Routes to configure on interface Routes []Route - // AddInterfacesDataToPodInfo is set to true for SF SwiftV2 scenario - AddInterfacesDataToPodInfo bool } type HostIPInfo struct { - Gateway string - PrimaryIP string - SecondaryIP string - Subnet string + Gateway string + PrimaryIP string + Subnet string } type IPConfigRequest struct { diff --git a/cns/api.go b/cns/api.go index acd423bd79..3894e9aff6 100644 --- a/cns/api.go +++ b/cns/api.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "net" "time" "github.com/Azure/azure-container-networking/cns/common" @@ -365,35 +364,3 @@ type EndpointRequest struct { HnsEndpointID string `json:"hnsEndpointID"` HostVethName string `json:"hostVethName"` } - -// GetEndpointResponse describes response from the The GetEndpoint API. -type GetEndpointResponse struct { - Response Response `json:"response"` - EndpointInfo EndpointInfo `json:"endpointInfo"` -} - -type EndpointInfo struct { - PodName string - PodNamespace string - IfnameToIPMap map[string]*IPInfo // key : interface name, value : IPInfo - HnsEndpointID string - HostVethName string -} -type IPInfo struct { - IPv4 []net.IPNet - IPv6 []net.IPNet -} - -type GetHTTPServiceDataResponse struct { - HTTPRestServiceData HTTPRestServiceData `json:"HTTPRestServiceData"` - Response Response `json:"response"` -} - -// HTTPRestServiceData represents in-memory CNS data in the debug API paths. -// PodInterfaceId is key and value is slice of Pod IP uuids in PodIPIDByPodInterfaceKey -// secondaryipid(uuid) is key for PodIPConfigState -type HTTPRestServiceData struct { - PodIPIDByPodInterfaceKey map[string][]string `json:"podIPIDByPodInterfaceKey"` - PodIPConfigState map[string]IPConfigurationStatus `json:"podIpConfigState"` - IPAMPoolMonitor IpamPoolMonitorStateSnapshot `json:"ipamPoolMonitor"` -} diff --git a/cns/client/client.go b/cns/client/client.go index 35cc03603b..2ee524cfb9 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -11,6 +11,7 @@ import ( "time" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/cns/types" "github.com/pkg/errors" ) @@ -562,7 +563,7 @@ func (c *Client) GetPodOrchestratorContext(ctx context.Context) (map[string][]st } // GetHTTPServiceData gets all public in-memory struct details for debugging purpose -func (c *Client) GetHTTPServiceData(ctx context.Context) (*cns.GetHTTPServiceDataResponse, error) { +func (c *Client) GetHTTPServiceData(ctx context.Context) (*restserver.GetHTTPServiceDataResponse, error) { u := c.routes[cns.PathDebugRestData] req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { @@ -582,7 +583,7 @@ func (c *Client) GetHTTPServiceData(ctx context.Context) (*cns.GetHTTPServiceDat if res.StatusCode != http.StatusOK { return nil, errors.Errorf("http response %d", res.StatusCode) } - var resp cns.GetHTTPServiceDataResponse + var resp restserver.GetHTTPServiceDataResponse err = json.NewDecoder(bytes.NewReader(b)).Decode(&resp) if err != nil { return nil, errors.Wrap(err, "failed to decode GetHTTPServiceDataResponse") @@ -1023,7 +1024,7 @@ func (c *Client) GetHomeAz(ctx context.Context) (*cns.GetHomeAzResponse, error) } // GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given EndpointID -func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*cns.GetEndpointResponse, error) { +func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) { // build the request u := c.routes[cns.EndpointAPI] uString := u.String() + endpointID @@ -1043,7 +1044,7 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*cns.GetEn return nil, errors.Errorf("http response %d", res.StatusCode) } - var response cns.GetEndpointResponse + var response restserver.GetEndpointResponse err = json.NewDecoder(res.Body).Decode(&response) if err != nil { return nil, errors.Wrap(err, "failed to decode GetEndpointResponse") diff --git a/cns/client/client_test.go b/cns/client/client_test.go index e7265a5483..bcadde14a7 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -7,22 +7,46 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "os" "sort" + "strconv" "testing" "time" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/common" + "github.com/Azure/azure-container-networking/cns/fakes" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/Azure/azure-container-networking/log" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + primaryIP = "10.0.0.5" + gatewayIP = "10.0.0.1" + subnetPrfixLength = 24 + dockerContainerType = cns.Docker + releasePercent = 150 + requestPercent = 50 + batchSize = 10 + initPoolSize = 10 + testpodname = "testpodname" + testpodnamespace = "testpodnamespace" ) var ( + svc *restserver.HTTPRestService + dnsServers = []string{"8.8.8.8", "8.8.4.4"} errBadRequest = errors.New("bad request") ) @@ -42,7 +66,89 @@ func (m *mockdo) Do(req *http.Request) (*http.Response, error) { }, m.errToReturn } +func addTestStateToRestServer(t *testing.T, secondaryIps []string) { + var ipConfig cns.IPConfiguration + ipConfig.DNSServers = dnsServers + ipConfig.GatewayIPAddress = gatewayIP + var ipSubnet cns.IPSubnet + ipSubnet.IPAddress = primaryIP + ipSubnet.PrefixLength = subnetPrfixLength + ipConfig.IPSubnet = ipSubnet + secondaryIPConfigs := make(map[string]cns.SecondaryIPConfig) + + for _, secIPAddress := range secondaryIps { + secIPConfig := cns.SecondaryIPConfig{ + IPAddress: secIPAddress, + NCVersion: -1, + } + ipID := uuid.New() + secondaryIPConfigs[ipID.String()] = secIPConfig + } + + req := &cns.CreateNetworkContainerRequest{ + NetworkContainerType: dockerContainerType, + NetworkContainerid: "testNcId1", + IPConfiguration: ipConfig, + SecondaryIPConfigs: secondaryIPConfigs, + // Set it as -1 to be same as default host version. + // It will allow secondary IPs status to be set as available. + Version: "-1", + } + + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) + if returnCode != 0 { + t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) + } + + _ = svc.IPAMPoolMonitor.Update(&v1alpha.NodeNetworkConfig{ + Spec: v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: 16, + IPsNotInUse: []string{"abc"}, + }, + Status: v1alpha.NodeNetworkConfigStatus{ + Scaler: v1alpha.Scaler{ + BatchSize: batchSize, + ReleaseThresholdPercent: releasePercent, + RequestThresholdPercent: requestPercent, + MaxIPCount: 250, + }, + NetworkContainers: []v1alpha.NetworkContainer{ + {}, + }, + }, + }) +} + +func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) { + var ( + resultIPnet net.IPNet + err error + ) + + // set result ipconfig from CNS Response Body + prefix := strconv.Itoa(int(resp.PodIpInfo.PodIPConfig.PrefixLength)) + ip, ipnet, err := net.ParseCIDR(resp.PodIpInfo.PodIPConfig.IPAddress + "/" + prefix) + if err != nil { + return resultIPnet, err //nolint:wrapcheck // ignore wrapping for test + } + + // construct ipnet for result + resultIPnet = net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + } + return resultIPnet, err //nolint:wrapcheck // we don't need error wrapping for tests +} + func TestMain(m *testing.M) { + var ( + info = &cns.SetOrchestratorTypeRequest{ + OrchestratorType: cns.KubernetesCRD, + } + body bytes.Buffer + res *http.Response + ) + tmpFileState, err := os.CreateTemp(os.TempDir(), "cns-*.json") tmpLogDir, err := os.MkdirTemp("", "cns-") fmt.Printf("logdir: %+v", tmpLogDir) @@ -63,11 +169,229 @@ func TestMain(m *testing.M) { } logger.InitLogger(logName, 0, 0, tmpLogDir+"/") + config := common.ServiceConfig{} + + httpRestService, err := restserver.NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, nil, nil, nil) + svc = httpRestService + httpRestService.Name = "cns-test-server" + fakeNNC := v1alpha.NodeNetworkConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: 16, + IPsNotInUse: []string{"abc"}, + }, + Status: v1alpha.NodeNetworkConfigStatus{ + Scaler: v1alpha.Scaler{ + BatchSize: 10, + ReleaseThresholdPercent: 150, + RequestThresholdPercent: 50, + MaxIPCount: 250, + }, + NetworkContainers: []v1alpha.NetworkContainer{ + { + ID: "nc1", + PrimaryIP: "10.0.0.11", + SubnetName: "sub1", + IPAssignments: []v1alpha.IPAssignment{ + { + Name: "ip1", + IP: "10.0.0.10", + }, + }, + DefaultGateway: "10.0.0.1", + SubnetAddressSpace: "10.0.0.0/24", + Version: 2, + }, + }, + }, + } + httpRestService.IPAMPoolMonitor = &fakes.MonitorFake{IPsNotInUseCount: 13, NodeNetworkConfig: &fakeNNC} + + if err != nil { + logger.Errorf("Failed to create CNS object, err:%v.\n", err) + return + } + + if httpRestService != nil { + err = httpRestService.Init(&config) + if err != nil { + logger.Errorf("Failed to initialize HttpService, err:%v.\n", err) + return + } + + err = httpRestService.Start(&config) + if err != nil { + logger.Errorf("Failed to start HttpService, err:%v.\n", err) + return + } + } + + if jsonErr := json.NewEncoder(&body).Encode(info); jsonErr != nil { + log.Errorf("encoding json failed with %v", jsonErr) + return + } + + httpc := &http.Client{} + setOrchURL := defaultBaseURL + cns.SetOrchestratorType + + res, err = httpc.Post(setOrchURL, "application/json", &body) //nolint:noctx // ignore for unit test + if err != nil { + fmt.Println(err) + } + defer res.Body.Close() + fmt.Println(res) exitCode := m.Run() os.Exit(exitCode) } +func TestCNSClientRequestAndRelease(t *testing.T) { + podName := testpodname + podNamespace := testpodnamespace + desiredIPAddress := primaryIP + ip := net.ParseIP(desiredIPAddress) + _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") + desired := net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + } + + secondaryIps := make([]string, 0) + secondaryIps = append(secondaryIps, desiredIPAddress) + cnsClient, _ := New("", 2*time.Hour) + + addTestStateToRestServer(t, secondaryIps) + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + orchestratorContext, err := json.Marshal(podInfo) + require.NoError(t, err) + + // no IP reservation found with that context, expect no failure. + err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) + require.NoError(t, err, "Release ip idempotent call failed") + + // request IP address + resp, err := cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) + require.NoError(t, err, "get IP from CNS failed") + + podIPInfo := resp.PodIpInfo + assert.Equal(t, primaryIP, podIPInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress, "PrimaryIP is not added as epected ipConfig") + assert.EqualValues(t, podIPInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength, subnetPrfixLength, "Primary IP Prefix length is not added as expected ipConfig") + + // validate DnsServer and Gateway Ip as the same configured for Primary IP + assert.Equal(t, dnsServers, podIPInfo.NetworkContainerPrimaryIPConfig.DNSServers, "DnsServer is not added as expected ipConfig") + assert.Equal(t, gatewayIP, podIPInfo.NetworkContainerPrimaryIPConfig.GatewayIPAddress, "Gateway is not added as expected ipConfig") + + resultIPnet, err := getIPNetFromResponse(resp) + require.NoError(t, err, "Get IPNetFromResponse failed") + + assert.Equal(t, desired, resultIPnet, "Desired result not matching actual result") + + // checking for assigned IP address and pod context printing before ReleaseIPAddress is called + ipaddresses, err := cnsClient.GetIPAddressesMatchingStates(context.TODO(), types.Assigned) + require.NoError(t, err, "Get assigned IP addresses failed") + + assert.Len(t, ipaddresses, 1, "Number of available IP addresses expected to be 1") + assert.Equal(t, desiredIPAddress, ipaddresses[0].IPAddress, "Available IP address does not match expected, address state") + assert.Equal(t, types.Assigned, ipaddresses[0].GetState(), "Available IP address does not match expected, address state") + + t.Log(ipaddresses) + + addresses := make([]string, len(ipaddresses)) + for i := range ipaddresses { + addresses[i] = ipaddresses[i].IPAddress + } + + // release requested IP address, expect success + err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{DesiredIPAddress: ipaddresses[0].IPAddress, OrchestratorContext: orchestratorContext}) + require.NoError(t, err, "Expected to not fail when releasing IP reservation found with context") +} + +func TestCNSClientPodContextApi(t *testing.T) { + desiredIPAddress := primaryIP + + secondaryIps := []string{desiredIPAddress} + cnsClient, _ := New("", 2*time.Second) + + addTestStateToRestServer(t, secondaryIps) + + podInfo := cns.NewPodInfo("some-guid-1", "abc-eth0", testpodname, testpodnamespace) + orchestratorContext, err := json.Marshal(podInfo) + require.NoError(t, err) + + // request IP address + _, err = cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) + require.NoError(t, err, "get IP from CNS failed") + + // test for pod ip by orch context map + podcontext, err := cnsClient.GetPodOrchestratorContext(context.TODO()) + require.NoError(t, err, "Get pod ip by orchestrator context failed") + assert.NotEmpty(t, podcontext, "Expected at least 1 entry in map for podcontext") + + t.Log(podcontext) + + // release requested IP address, expect success + err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) + require.NoError(t, err, "Expected to not fail when releasing IP reservation found with context") +} + +func TestCNSClientDebugAPI(t *testing.T) { + podName := testpodname + podNamespace := testpodnamespace + desiredIPAddress := primaryIP + + secondaryIps := []string{desiredIPAddress} + cnsClient, _ := New("", 2*time.Hour) + + addTestStateToRestServer(t, secondaryIps) + + podInfo := cns.NewPodInfo("some-guid-1", "abc-eth0", podName, podNamespace) + orchestratorContext, err := json.Marshal(podInfo) + require.NoError(t, err) + + // request IP address + _, err1 := cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) + require.NoError(t, err1, "get IP from CNS failed") + + // test for debug api/cmd to get inmemory data from HTTPRestService + inmemory, err := cnsClient.GetHTTPServiceData(context.TODO()) + require.NoError(t, err, "Get in-memory http REST Struct failed") + + assert.NotEmpty(t, inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey, "OrchestratorContext map is expected but not returned") + + // testing Pod IP Configuration Status values set for test + podConfig := inmemory.HTTPRestServiceData.PodIPConfigState + for _, v := range podConfig { + assert.Equal(t, "10.0.0.5", v.IPAddress, "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) + assert.Equal(t, types.Assigned, v.GetState(), "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) + assert.Equal(t, "testNcId1", v.NCID, "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) + } + assert.NotEmpty(t, inmemory.HTTPRestServiceData.PodIPConfigState, "PodIpConfigState with at least 1 entry expected") + + testIpamPoolMonitor := inmemory.HTTPRestServiceData.IPAMPoolMonitor + assert.EqualValues(t, 5, testIpamPoolMonitor.MinimumFreeIps, "IPAMPoolMonitor state is not reflecting the initial set values") + assert.EqualValues(t, 15, testIpamPoolMonitor.MaximumFreeIps, "IPAMPoolMonitor state is not reflecting the initial set values") + assert.EqualValues(t, 13, testIpamPoolMonitor.UpdatingIpsNotInUseCount, "IPAMPoolMonitor state is not reflecting the initial set values") + + // check for cached NNC Spec struct values + assert.EqualValues(t, 16, testIpamPoolMonitor.CachedNNC.Spec.RequestedIPCount, "IPAMPoolMonitor cached NNC Spec is not reflecting the initial set values") + assert.Len(t, testIpamPoolMonitor.CachedNNC.Spec.IPsNotInUse, 1, "IPAMPoolMonitor cached NNC Spec is not reflecting the initial set values") + + // check for cached NNC Status struct values + assert.EqualValues(t, 10, testIpamPoolMonitor.CachedNNC.Status.Scaler.BatchSize, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") + assert.EqualValues(t, 150, testIpamPoolMonitor.CachedNNC.Status.Scaler.ReleaseThresholdPercent, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") + assert.EqualValues(t, 50, testIpamPoolMonitor.CachedNNC.Status.Scaler.RequestThresholdPercent, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") + assert.Len(t, testIpamPoolMonitor.CachedNNC.Status.NetworkContainers, 1, "Expected only one Network Container in the list") + + t.Logf("In-memory Data: ") + for i := range inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey { + t.Logf("PodIPIDByOrchestratorContext: %+v", inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey[i]) + } + t.Logf("PodIPConfigState: %+v", inmemory.HTTPRestServiceData.PodIPConfigState) + t.Logf("IPAMPoolMonitor: %+v", inmemory.HTTPRestServiceData.IPAMPoolMonitor) +} + func TestNew(t *testing.T) { fqdnBaseURL := "http://testinstance.centraluseuap.cloudapp.azure.com" fqdnWithPortBaseURL := fqdnBaseURL + ":10090" @@ -322,13 +646,13 @@ func TestGetAllNetworkContainers(t *testing.T) { } orchestratorContext, err := json.Marshal(tt.podInfo) - assert.NoError(t, err, "marshaling orchestrator context failed") + require.NoError(t, err, "marshaling orchestrator context failed") got, err := client.GetAllNetworkContainers(tt.ctx, orchestratorContext) if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -453,13 +777,13 @@ func TestGetNetworkConfiguration(t *testing.T) { } orchestratorContext, err := json.Marshal(tt.podInfo) - assert.NoError(t, err, "marshaling orchestrator context failed") + require.NoError(t, err, "marshaling orchestrator context failed") got, err := client.GetNetworkContainer(tt.ctx, orchestratorContext) if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -567,7 +891,7 @@ func TestCreateHostNCApipaEndpoint(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -1254,7 +1578,7 @@ func TestDeleteHostNCApipaEndpoint(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } }) } @@ -1381,7 +1705,7 @@ func TestRequestIPAddress(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -1500,7 +1824,7 @@ func TestReleaseIPAddress(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } }) } @@ -1657,7 +1981,7 @@ func TestRequestIPs(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -1804,7 +2128,7 @@ func TestReleaseIPs(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } }) } @@ -1921,7 +2245,7 @@ func TestGetIPAddressesMatchingStates(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -2023,7 +2347,7 @@ func TestGetPodOrchestratorContext(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -2037,7 +2361,7 @@ func TestGetHTTPServiceData(t *testing.T) { ctx context.Context mockdo *mockdo routes map[string]url.URL - want *cns.GetHTTPServiceDataResponse + want *restserver.GetHTTPServiceDataResponse wantErr bool }{ { @@ -2045,11 +2369,11 @@ func TestGetHTTPServiceData(t *testing.T) { ctx: context.TODO(), mockdo: &mockdo{ errToReturn: nil, - objToReturn: &cns.GetHTTPServiceDataResponse{}, + objToReturn: &restserver.GetHTTPServiceDataResponse{}, httpStatusCodeToReturn: http.StatusOK, }, routes: emptyRoutes, - want: &cns.GetHTTPServiceDataResponse{}, + want: &restserver.GetHTTPServiceDataResponse{}, wantErr: false, }, { @@ -2069,7 +2393,7 @@ func TestGetHTTPServiceData(t *testing.T) { ctx: context.TODO(), mockdo: &mockdo{ errToReturn: nil, - objToReturn: []cns.GetHTTPServiceDataResponse{}, + objToReturn: []restserver.GetHTTPServiceDataResponse{}, httpStatusCodeToReturn: http.StatusOK, }, routes: emptyRoutes, @@ -2093,8 +2417,8 @@ func TestGetHTTPServiceData(t *testing.T) { ctx: context.TODO(), mockdo: &mockdo{ errToReturn: nil, - objToReturn: &cns.GetHTTPServiceDataResponse{ - Response: cns.Response{ + objToReturn: &restserver.GetHTTPServiceDataResponse{ + Response: restserver.Response{ ReturnCode: types.UnsupportedNetworkType, }, }, @@ -2123,7 +2447,7 @@ func TestGetHTTPServiceData(t *testing.T) { if tt.wantErr { assert.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } assert.Equal(t, tt.want, got) }) @@ -2286,7 +2610,7 @@ func TestGetAllNCsFromCns(t *testing.T) { } got, err := client.GetAllNCsFromCns(context.TODO()) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, exp.Response.ReturnCode, got.Response.ReturnCode) } diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index b31ae0c7f0..44da9c3e87 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -37,6 +37,7 @@ type CNSConfig struct { EnablePprof bool EnableStateMigration bool EnableSubnetScarcity bool + EnableSwiftV2 bool InitializeFromCNI bool KeyVaultSettings KeyVaultSettings MSISettings MSISettings @@ -219,5 +220,5 @@ func SetCNSConfigDefaults(config *CNSConfig) { if config.AsyncPodDeletePath == "" { config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs" } - config.WatchPods = config.EnableIPAMv2 || config.SWIFTV2Mode == K8sSWIFTV2 + config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2 } diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 64c958dd36..cc24b7917a 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -27,9 +27,8 @@ const ( ExtHnsNetworkGwAddress = "192.168.255.1" // HNS network types - hnsL2Bridge = "l2bridge" - hnsL2Tunnel = "l2tunnel" - hnsTransparent = "transparent" + hnsL2Bridge = "l2bridge" + hnsL2Tunnel = "l2tunnel" // hcnSchemaVersionMajor indicates major version number for hcn schema hcnSchemaVersionMajor = 2 @@ -138,7 +137,7 @@ func CreateDefaultExtNetwork(networkType string) error { return nil } - if networkType != hnsL2Bridge && networkType != hnsL2Tunnel && networkType != hnsTransparent { + if networkType != hnsL2Bridge && networkType != hnsL2Tunnel { return fmt.Errorf("Invalid hns network type %s", networkType) } diff --git a/cns/restserver/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go similarity index 99% rename from cns/restserver/k8sSwiftV2.go rename to cns/middlewares/k8sSwiftV2.go index ed11020b88..a9721c3995 100644 --- a/cns/restserver/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -1,4 +1,4 @@ -package restserver +package middlewares import ( "context" diff --git a/cns/restserver/k8sSwiftV2_test.go b/cns/middlewares/k8sSwiftV2_test.go similarity index 93% rename from cns/restserver/k8sSwiftV2_test.go rename to cns/middlewares/k8sSwiftV2_test.go index 70d4e0359d..856c4cd521 100644 --- a/cns/restserver/k8sSwiftV2_test.go +++ b/cns/middlewares/k8sSwiftV2_test.go @@ -1,4 +1,4 @@ -package restserver +package middlewares import ( "context" @@ -7,15 +7,31 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "gotest.tools/v3/assert" ) var ( + testPod1GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod1Info = cns.NewPodInfo("898fb8-eth0", testPod1GUID, "testpod1", "testpod1namespace") + + testPod2GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod2Info = cns.NewPodInfo("b21e1e-eth0", testPod2GUID, "testpod2", "testpod2namespace") + + testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" + testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") + + testPod4GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" testPod4Info = cns.NewPodInfo("b21e1e-eth0", testPod4GUID, "testpod4", "testpod4namespace") ) +func TestMain(m *testing.M) { + logger.InitLogger("testlogs", 0, 0, "./") + m.Run() +} + func TestIPConfigsRequestHandlerWrapperSuccess(t *testing.T) { middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") diff --git a/cns/restserver/SFSwiftV2.go b/cns/restserver/SFSwiftV2.go deleted file mode 100644 index d66ef93806..0000000000 --- a/cns/restserver/SFSwiftV2.go +++ /dev/null @@ -1,82 +0,0 @@ -package restserver - -import ( - "context" - "fmt" - - "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/client" - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/types" - "github.com/pkg/errors" -) - -type SFSWIFTv2Middleware struct { - CnsClient *client.Client -} - -// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP config requests for SF standalone scenario. This function wraps the default SWIFT request -// and release IP configs handlers. -func (m *SFSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(_, _ cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { - return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - // unmarshal & retrieve podInfo from OrchestratorContext - podInfo, err := cns.NewPodInfoFromIPConfigsRequest(req) - ipConfigsResp := &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.Success, - }, - PodIPInfo: []cns.PodIpInfo{}, - } - if err != nil { - ipConfigsResp.Response.ReturnCode = types.UnexpectedError - return ipConfigsResp, errors.Wrapf(err, "Failed to receive PodInfo after unmarshalling from IPConfigsRequest %v", req) - } - - // SwiftV2-SF will always request for secondaryInterfaces for a pod - req.SecondaryInterfacesExist = true - logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) - - // get the IPConfig for swiftv2 SF scenario by calling into cns getNC api - SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) - } - ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo) - return ipConfigsResp, nil - } -} - -// getIPConfig returns the pod's SWIFT V2 IP configuration. -func (m *SFSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { - orchestratorContext, err := podInfo.OrchestratorContext() - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("error getting orchestrator context from PodInfo %w", err) - } - // call getNC via CNSClient - resp, err := m.CnsClient.GetNetworkContainer(ctx, orchestratorContext) - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("error getNetworkContainerByOrchestrator Context %w", err) - } - - // Check if the ncstate/ipconfig ready. If one of the fields is empty, return error - if resp.IPConfiguration.IPSubnet.IPAddress == "" || resp.NetworkInterfaceInfo.MACAddress == "" || resp.NetworkContainerID == "" || resp.IPConfiguration.GatewayIPAddress == "" { - return cns.PodIpInfo{}, fmt.Errorf("one of the fields for GetNCResponse is empty for given NC: %+v", resp) //nolint:goerr113 // return error - } - logger.Debugf("[SWIFTv2-SF] NetworkContainerResponse for pod %s is : %+v", podInfo.Name(), resp) - - podIPInfo := cns.PodIpInfo{ - PodIPConfig: resp.IPConfiguration.IPSubnet, - MacAddress: resp.NetworkInterfaceInfo.MACAddress, - NICType: resp.NetworkInterfaceInfo.NICType, - SkipDefaultRoutes: false, - NetworkContainerPrimaryIPConfig: resp.IPConfiguration, - AddInterfacesDataToPodInfo: true, - } - return podIPInfo, nil -} diff --git a/cns/restserver/cnsclient_test.go b/cns/restserver/cnsclient_test.go deleted file mode 100644 index 10190f887c..0000000000 --- a/cns/restserver/cnsclient_test.go +++ /dev/null @@ -1,298 +0,0 @@ -package restserver - -import ( - "context" - "encoding/json" - "net" - "strconv" - "testing" - "time" - - "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/client" - "github.com/Azure/azure-container-networking/cns/common" - "github.com/Azure/azure-container-networking/cns/fakes" - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/types" - "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - podnametest = "testpodname" - podnamespacetest = "testpodnamespace" - ncid = "440515e4-bf48-4997-8e4c-d650d29b462d" -) - -var dnsServers = []string{"8.8.8.8", "8.8.4.4"} - -func setUpRestserver() { - config := common.ServiceConfig{} - - httpRestService, err := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, nil, nil, nil) - httpRestService.Name = "cns-test-server" - fakeNNC := v1alpha.NodeNetworkConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: v1alpha.NodeNetworkConfigSpec{ - RequestedIPCount: 16, - IPsNotInUse: []string{"abc"}, - }, - Status: v1alpha.NodeNetworkConfigStatus{ - Scaler: v1alpha.Scaler{ - BatchSize: 10, - ReleaseThresholdPercent: 150, - RequestThresholdPercent: 50, - MaxIPCount: 250, - }, - NetworkContainers: []v1alpha.NetworkContainer{ - { - ID: "nc1", - PrimaryIP: "10.0.0.11", - SubnetName: "sub1", - IPAssignments: []v1alpha.IPAssignment{ - { - Name: "ip1", - IP: "10.0.0.10", - }, - }, - DefaultGateway: "10.0.0.1", - SubnetAddressSpace: "10.0.0.0/24", - Version: 2, - }, - }, - }, - } - httpRestService.IPAMPoolMonitor = &fakes.MonitorFake{IPsNotInUseCount: 13, NodeNetworkConfig: &fakeNNC} - - if err != nil { - logger.Errorf("Failed to create CNS object, err:%v.\n", err) - return - } - - svc = httpRestService - svc = service.(*HTTPRestService) - svc.state.OrchestratorType = cns.KubernetesCRD - svc.IPAMPoolMonitor = &fakes.MonitorFake{IPsNotInUseCount: 13, NodeNetworkConfig: &fakeNNC} -} - -func TestCNSClientRequestAndRelease(t *testing.T) { - desiredIPAddress := primaryIP - ip := net.ParseIP(desiredIPAddress) - _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") - desired := net.IPNet{ - IP: ip, - Mask: ipnet.Mask, - } - - secondaryIps := make([]string, 0) - secondaryIps = append(secondaryIps, desiredIPAddress) - cnsClient, _ := client.New("", 2*time.Hour) - setUpRestserver() - addTestStateToRestServer(t, secondaryIps) - - podInfo := cns.NewPodInfo("some-guid-1", "abc-eth0", podnametest, podnamespacetest) - orchestratorContext, err := json.Marshal(podInfo) - require.NoError(t, err) - - // no IP reservation found with that context, expect no failure. - err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) - require.NoError(t, err, "Release ip idempotent call failed") - - // request IP address - resp, err := cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) - require.NoError(t, err, "get IP from CNS failed") - - podIPInfo := resp.PodIpInfo - assert.Equal(t, primaryIP, podIPInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress, "PrimaryIP is not added as epected ipConfig") - assert.EqualValues(t, podIPInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength, subnetPrfixLength, "Primary IP Prefix length is not added as expected ipConfig") - - // validate DnsServer and Gateway Ip as the same configured for Primary IP - assert.Equal(t, dnsServers, podIPInfo.NetworkContainerPrimaryIPConfig.DNSServers, "DnsServer is not added as expected ipConfig") - assert.Equal(t, gatewayIP, podIPInfo.NetworkContainerPrimaryIPConfig.GatewayIPAddress, "Gateway is not added as expected ipConfig") - - resultIPnet, err := getIPNetFromResponse(resp) - require.NoError(t, err, "getIPNetFromResponse failed") - - assert.Equal(t, desired, resultIPnet, "Desired result not matching actual result") - - // checking for assigned IP address and pod context printing before ReleaseIPAddress is called - ipaddresses, err := cnsClient.GetIPAddressesMatchingStates(context.TODO(), types.Assigned) - require.NoError(t, err, "Get assigned IP addresses failed") - - assert.Len(t, ipaddresses, 1, "Number of available IP addresses expected to be 1") - assert.Equal(t, desiredIPAddress, ipaddresses[0].IPAddress, "Available IP address does not match expected, address state") - assert.Equal(t, types.Assigned, ipaddresses[0].GetState(), "Available IP address does not match expected, address state") - - t.Log(ipaddresses) - - addresses := make([]string, len(ipaddresses)) - for i := range ipaddresses { - addresses[i] = ipaddresses[i].IPAddress - } - - // release requested IP address, expect success - err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{DesiredIPAddress: ipaddresses[0].IPAddress, OrchestratorContext: orchestratorContext}) - require.NoError(t, err, "Expected to not fail when releasing IP reservation found with context") - service.Stop() -} - -func TestCNSClientPodContextApi(t *testing.T) { - desiredIPAddress := primaryIP - secondaryIps := []string{desiredIPAddress} - cnsClient, _ := client.New("", 2*time.Second) - - setUpRestserver() - addTestStateToRestServer(t, secondaryIps) - - podInfo := cns.NewPodInfo("some-guid-1", "abc-eth0", podnametest, podnamespacetest) - orchestratorContext, err := json.Marshal(podInfo) - require.NoError(t, err) - - // request IP address - _, err = cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) - require.NoError(t, err, "get IP from CNS failed") - - // test for pod ip by orch context map - podcontext, err := cnsClient.GetPodOrchestratorContext(context.TODO()) - require.NoError(t, err, "Get pod ip by orchestrator context failed") - assert.NotEmpty(t, podcontext, "Expected at least 1 entry in map for podcontext") - t.Log(podcontext) - - // release requested IP address, expect success - err = cnsClient.ReleaseIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) - require.NoError(t, err, "Expected to not fail when releasing IP reservation found with context") - service.Stop() -} - -func TestCNSClientDebugAPI(t *testing.T) { - desiredIPAddress := primaryIP - - secondaryIPs := []string{desiredIPAddress} - cnsClient, _ := client.New("", 2*time.Hour) - - setUpRestserver() - addTestStateToRestServer(t, secondaryIPs) - - podInfo := cns.NewPodInfo("some-guid-1", "abc-eth0", podnametest, podnamespacetest) - orchestratorContext, err := json.Marshal(podInfo) - require.NoError(t, err) - - // request IP address - _, err1 := cnsClient.RequestIPAddress(context.TODO(), cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) - require.NoError(t, err1, "get IP from CNS failed") - - // test for debug api/cmd to get inmemory data from HTTPRestService - inmemory, err := cnsClient.GetHTTPServiceData(context.TODO()) - require.NoError(t, err, "Get in-memory http REST Struct failed") - assert.NotEmpty(t, inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey, "OrchestratorContext map is expected but not returned") - - // testing Pod IP Configuration Status values set for test - podConfig := inmemory.HTTPRestServiceData.PodIPConfigState - for _, v := range podConfig { - assert.Equal(t, primaryIP, v.IPAddress, "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) - assert.Equal(t, types.Assigned, v.GetState(), "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) - assert.Equal(t, ncid, v.NCID, "Not the expected set values for testing IPConfigurationStatus, %+v", podConfig) - } - assert.NotEmpty(t, inmemory.HTTPRestServiceData.PodIPConfigState, "PodIpConfigState with at least 1 entry expected") - - testIpamPoolMonitor := inmemory.HTTPRestServiceData.IPAMPoolMonitor - assert.EqualValues(t, 5, testIpamPoolMonitor.MinimumFreeIps, "IPAMPoolMonitor state is not reflecting the initial set values") - assert.EqualValues(t, 15, testIpamPoolMonitor.MaximumFreeIps, "IPAMPoolMonitor state is not reflecting the initial set values") - assert.EqualValues(t, 13, testIpamPoolMonitor.UpdatingIpsNotInUseCount, "IPAMPoolMonitor state is not reflecting the initial set values") - - // check for cached NNC Spec struct values - assert.EqualValues(t, 16, testIpamPoolMonitor.CachedNNC.Spec.RequestedIPCount, "IPAMPoolMonitor cached NNC Spec is not reflecting the initial set values") - assert.Len(t, testIpamPoolMonitor.CachedNNC.Spec.IPsNotInUse, 1, "IPAMPoolMonitor cached NNC Spec is not reflecting the initial set values") - - // check for cached NNC Status struct values - assert.EqualValues(t, 10, testIpamPoolMonitor.CachedNNC.Status.Scaler.BatchSize, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") - assert.EqualValues(t, 150, testIpamPoolMonitor.CachedNNC.Status.Scaler.ReleaseThresholdPercent, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") - assert.EqualValues(t, 50, testIpamPoolMonitor.CachedNNC.Status.Scaler.RequestThresholdPercent, "IPAMPoolMonitor cached NNC Status is not reflecting the initial set values") - assert.Len(t, testIpamPoolMonitor.CachedNNC.Status.NetworkContainers, 1, "Expected only one Network Container in the list") - - t.Logf("In-memory Data: ") - for i := range inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey { - t.Logf("PodIPIDByOrchestratorContext: %+v", inmemory.HTTPRestServiceData.PodIPIDByPodInterfaceKey[i]) - } - t.Logf("PodIPConfigState: %+v", inmemory.HTTPRestServiceData.PodIPConfigState) - t.Logf("IPAMPoolMonitor: %+v", inmemory.HTTPRestServiceData.IPAMPoolMonitor) - service.Stop() -} - -func addTestStateToRestServer(t *testing.T, secondaryIps []string) { - var ipConfig cns.IPConfiguration - ipConfig.DNSServers = dnsServers - ipConfig.GatewayIPAddress = gatewayIP - var ipSubnet cns.IPSubnet - ipSubnet.IPAddress = primaryIP - ipSubnet.PrefixLength = subnetPrfixLength - ipConfig.IPSubnet = ipSubnet - secondaryIPConfigs := make(map[string]cns.SecondaryIPConfig) - - for _, secIPAddress := range secondaryIps { - secIPConfig := cns.SecondaryIPConfig{ - IPAddress: secIPAddress, - NCVersion: -1, - } - ipID := uuid.New() - secondaryIPConfigs[ipID.String()] = secIPConfig - } - - req := &cns.CreateNetworkContainerRequest{ - NetworkContainerType: dockerContainerType, - NetworkContainerid: ncid, - IPConfiguration: ipConfig, - SecondaryIPConfigs: secondaryIPConfigs, - // Set it as -1 to be same as default host version. - // It will allow secondary IPs status to be set as available. - Version: "-1", - } - - returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) - if returnCode != 0 { - t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) - } - - _ = svc.IPAMPoolMonitor.Update(&v1alpha.NodeNetworkConfig{ - Spec: v1alpha.NodeNetworkConfigSpec{ - RequestedIPCount: 16, - IPsNotInUse: []string{"abc"}, - }, - Status: v1alpha.NodeNetworkConfigStatus{ - Scaler: v1alpha.Scaler{ - BatchSize: batchSize, - ReleaseThresholdPercent: 150, - RequestThresholdPercent: 50, - MaxIPCount: 250, - }, - NetworkContainers: []v1alpha.NetworkContainer{ - {}, - }, - }, - }) -} - -func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) { - var ( - resultIPnet net.IPNet - err error - ) - - // set result ipconfig from CNS Response Body - prefix := strconv.Itoa(int(resp.PodIpInfo.PodIPConfig.PrefixLength)) - ip, ipnet, err := net.ParseCIDR(resp.PodIpInfo.PodIPConfig.IPAddress + "/" + prefix) - if err != nil { - return resultIPnet, err //nolint:wrapcheck // we don't need error wrapping for tests - } - - // construct ipnet for result - resultIPnet = net.IPNet{ - IP: ip, - Mask: ipnet.Mask, - } - return resultIPnet, err //nolint:wrapcheck // we don't need error wrapping for tests -} diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index bde4be6c82..904cc33974 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -521,7 +521,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns. // For now only RequestController uses this API which will be initialized only for AKS scenario. // Validate ContainerType is set as Docker - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes && service.state.OrchestratorType != cns.ServiceFabric { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { logger.Errorf("[Azure CNS] Error. Unsupported OrchestratorType: %s", service.state.OrchestratorType) return types.UnsupportedOrchestratorType } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index fc7be99059..fa4b31062f 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -36,6 +36,7 @@ const ( // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ @@ -45,9 +46,11 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, }, errors.New("failed to validate ip config request") } + // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) - podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest) //nolint:contextcheck // to refactor later + + podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest) //nolint:contextcheck // appease linter for revert PR if err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -57,6 +60,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context PodIPInfo: podIPInfo, }, err } + // record a pod assigned an IP defer func() { // observe IP assignment wait time @@ -64,6 +68,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context ipAssignmentLatency.Observe(since.Seconds()) } }() + // Check if http rest service managed endpoint state is set if service.Options[common.OptManageEndpointState] == true { err = service.updateEndpointState(ipconfigsRequest, podInfo, podIPInfo) @@ -77,6 +82,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, err } } + return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.Success, @@ -187,54 +193,11 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r return } - if ipConfigsResp.PodIPInfo[0].AddInterfacesDataToPodInfo { - ipConfigsResp, err = service.updatePodInfoWithInterfaces(r.Context(), ipConfigsResp) - if err != nil { - w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) - err = service.Listener.Encode(w, &ipConfigsResp) - logger.ResponseEx(service.Name+operationName, ipconfigsRequest, ipConfigsResp, ipConfigsResp.Response.ReturnCode, err) - return - } - } - w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) err = service.Listener.Encode(w, &ipConfigsResp) logger.ResponseEx(service.Name+operationName, ipconfigsRequest, ipConfigsResp, ipConfigsResp.Response.ReturnCode, err) } -func (service *HTTPRestService) updatePodInfoWithInterfaces(ctx context.Context, ipconfigResponse *cns.IPConfigsResponse) (*cns.IPConfigsResponse, error) { - podIPInfoList := make([]cns.PodIpInfo, 0, len(ipconfigResponse.PodIPInfo)) - for i := range ipconfigResponse.PodIPInfo { - // populating podIpInfo with primary & secondary interface info & updating IpConfigsResponse - hostPrimaryInterface, err := service.getPrimaryHostInterface(ctx) - if err != nil { - return &cns.IPConfigsResponse{}, err - } - - hostSecondaryInterface, err := service.getSecondaryHostInterface(ctx, ipconfigResponse.PodIPInfo[i].MacAddress) - if err != nil { - return &cns.IPConfigsResponse{}, err - } - - ipconfigResponse.PodIPInfo[i].HostPrimaryIPInfo = cns.HostIPInfo{ - Gateway: hostPrimaryInterface.Gateway, - PrimaryIP: hostPrimaryInterface.PrimaryIP, - Subnet: hostPrimaryInterface.Subnet, - } - - ipconfigResponse.PodIPInfo[i].HostSecondaryIPInfo = cns.HostIPInfo{ - Gateway: hostSecondaryInterface.Gateway, - SecondaryIP: hostSecondaryInterface.SecondaryIPs[0], - Subnet: hostSecondaryInterface.Subnet, - } - - podIPInfoList = append(podIPInfoList, ipconfigResponse.PodIPInfo[i]) - - } - ipconfigResponse.PodIPInfo = podIPInfoList - return ipconfigResponse, nil -} - func (service *HTTPRestService) updateEndpointState(ipconfigsRequest cns.IPConfigsRequest, podInfo cns.PodInfo, podIPInfo []cns.PodIpInfo) error { if service.EndpointStateStore == nil { return ErrStoreEmpty @@ -614,8 +577,8 @@ func (service *HTTPRestService) handleDebugRestData(w http.ResponseWriter, r *ht http.Error(w, "not ready", http.StatusServiceUnavailable) return } - resp := cns.GetHTTPServiceDataResponse{ - HTTPRestServiceData: cns.HTTPRestServiceData{ + resp := GetHTTPServiceDataResponse{ + HTTPRestServiceData: HTTPRestServiceData{ PodIPIDByPodInterfaceKey: service.PodIPIDByPodInterfaceKey, PodIPConfigState: service.PodIPConfigState, IPAMPoolMonitor: service.IPAMPoolMonitor.GetStateSnapshot(), diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 6b1f3234f8..1c545bc5da 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -10,14 +10,12 @@ import ( "net/netip" "strconv" "testing" - "time" "github.com/Azure/azure-container-networking/cns" - cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/fakes" - "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" @@ -1487,7 +1485,7 @@ func TestIPAMFailToReleasePartialIPsInPool(t *testing.T) { if err != nil { t.Fatalf("Expected to not fail adding empty NC to state: %+v", err) } - // remove the IP from the ipconfig map so that it throws an error when trying to release one of the IPs + // remove the IP from the from the ipconfig map so that it throws an error when trying to release one of the IPs delete(svc.PodIPConfigState, testStatev6.ID) err = svc.releaseIPConfigs(testPod1Info) @@ -1517,7 +1515,7 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { if err != nil { t.Fatalf("Expected to not fail adding empty NC to state: %+v", err) } - // remove the IP from the ipconfig map so that it throws an error when trying to release one of the IPs + // remove the IP from the from the ipconfig map so that it throws an error when trying to release one of the IPs delete(svc.PodIPConfigState, testStatev6.ID) req := cns.IPConfigsRequest{ @@ -1538,7 +1536,7 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { func TestIPAMReleaseSWIFTV2PodIPSuccess(t *testing.T) { svc := getTestService() - middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + middleware := middlewares.K8sSWIFTv2Middleware{Cli: mock.NewClient()} svc.AttachIPConfigsHandlerMiddleware(&middleware) t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") @@ -1589,7 +1587,7 @@ func TestIPAMReleaseSWIFTV2PodIPSuccess(t *testing.T) { func TestIPAMGetK8sSWIFTv2IPSuccess(t *testing.T) { svc := getTestService() - middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + middleware := middlewares.K8sSWIFTv2Middleware{Cli: mock.NewClient()} svc.AttachIPConfigsHandlerMiddleware(&middleware) t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") @@ -1652,7 +1650,7 @@ func TestIPAMGetK8sSWIFTv2IPSuccess(t *testing.T) { func TestIPAMGetK8sSWIFTv2IPFailure(t *testing.T) { svc := getTestService() - middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + middleware := middlewares.K8sSWIFTv2Middleware{Cli: mock.NewClient()} svc.AttachIPConfigsHandlerMiddleware(&middleware) ncStates := []ncState{ { @@ -1719,27 +1717,3 @@ func TestIPAMGetK8sSWIFTv2IPFailure(t *testing.T) { t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") } } - -func TestValidateSFIPConfigsRequestFailure(t *testing.T) { - svc := getTestService() - cnsClient, err := cnsclient.New("", 15*time.Second) - if err != nil { - logger.Errorf("Failed to init cnsclient, err:%v.\n", err) - return - } - middleware := SFSWIFTv2Middleware{CnsClient: cnsClient} - svc.AttachIPConfigsHandlerMiddleware(&middleware) - - // Fail to unmarshal pod info test - failReq := cns.IPConfigsRequest{ - PodInterfaceID: testPod1Info.InterfaceID(), - InfraContainerID: testPod1Info.InfraContainerID(), - } - failReq.OrchestratorContext = []byte("invalid") - - wrappedHandler := svc.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(svc.requestIPConfigHandlerHelper, svc.releaseIPConfigHandlerHelper) - resp, err := wrappedHandler(context.TODO(), failReq) - - assert.NotEmpty(t, err) - assert.Equal(t, types.UnexpectedError, resp.Response.ReturnCode) -} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 6e1f72da73..e67019205a 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -103,6 +103,19 @@ type IPInfo struct { IPv6 []net.IPNet } +type GetHTTPServiceDataResponse struct { + HTTPRestServiceData HTTPRestServiceData `json:"HTTPRestServiceData"` + Response Response `json:"Response"` +} + +// HTTPRestServiceData represents in-memory CNS data in the debug API paths. +// TODO: add json tags for this struct as per linter suggestion, ignored for now as part of revert-PR +type HTTPRestServiceData struct { //nolint:musttag // not tagging struct for revert-PR + PodIPIDByPodInterfaceKey map[string][]string // PodInterfaceId is key and value is slice of Pod IP uuids. + PodIPConfigState map[string]cns.IPConfigurationStatus // secondaryipid(uuid) is key + IPAMPoolMonitor cns.IpamPoolMonitorStateSnapshot +} + type Response struct { ReturnCode types.ResponseCode Message string @@ -136,7 +149,6 @@ type httpRestServiceState struct { TimeStamp time.Time joinedNetworks map[string]struct{} primaryInterface *wireserver.InterfaceInfo - secondaryInterface *wireserver.InterfaceInfo } type networkInfo struct { diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 6c90000587..b65e440a54 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -756,12 +756,12 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) { - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes && service.state.OrchestratorType != cns.ServiceFabric { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" } if ipConfigsRequest.OrchestratorContext == nil { - return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchestratorContext is not set in the req: %+v", ipConfigsRequest) + return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) } // retrieve podinfo from orchestrator context @@ -790,24 +790,6 @@ func (service *HTTPRestService) getPrimaryHostInterface(ctx context.Context) (*w return service.state.primaryInterface, nil } -// getSecondaryHostInterface returns the cached InterfaceInfo, if available, otherwise -// queries the IMDS to get the secondary interface info and caches it in the server-state before returning the result. -func (service *HTTPRestService) getSecondaryHostInterface(ctx context.Context, macAddress string) (*wireserver.InterfaceInfo, error) { - if service.state.secondaryInterface == nil { - res, err := service.wscli.GetInterfaces(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to get interfaces from wireserver client") - } - secondary, err := wireserver.GetSecondaryInterfaceFromResult(res, macAddress) - if err != nil { - return nil, errors.Wrap(err, "failed to get secondary interface from wireserver client") - } - - service.state.secondaryInterface = secondary - } - return service.state.secondaryInterface, nil -} - //nolint:gocritic // ignore hugeParam pls func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus cns.IPConfigurationStatus, podIPInfo *cns.PodIpInfo) error { ncStatus, exists := service.state.ContainerStatus[ipConfigStatus.NCID] diff --git a/cns/service/main.go b/cns/service/main.go index 8958ae2021..5a0bac7dc0 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -39,6 +39,7 @@ import ( nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" @@ -800,16 +801,6 @@ func main() { if platform.HasMellanoxAdapter() { go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs) } - // if swiftv2 scenario is enabled, we need to initialize the Service Fabric (standalone) swiftv2 middleware to process IP configs requests - if cnsconfig.SWIFTV2Mode == configuration.SFSWIFTV2 { - cnsClient, err := cnsclient.New("", cnsReqTimeout) //nolint:govet // shadow ok as function returns in above errs - if err != nil { - logger.Errorf("Failed to init cnsclient, err:%v.\n", err) - return - } - swiftV2Middleware := &restserver.SFSWIFTv2Middleware{CnsClient: cnsClient} - httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware) - } } // Initialze state in if CNS is running in CRD mode @@ -1229,7 +1220,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // check the Node labels for Swift V2 if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { - cnsconfig.SWIFTV2Mode = configuration.K8sSWIFTV2 + cnsconfig.EnableSwiftV2 = true cnsconfig.WatchPods = true if nodeInfoErr := createOrUpdateNodeInfoCRD(ctx, kubeConfig, node); nodeInfoErr != nil { return errors.Wrap(nodeInfoErr, "error creating or updating nodeinfo crd") @@ -1413,13 +1404,22 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } - if cnsconfig.SWIFTV2Mode == configuration.K8sSWIFTV2 { + if cnsconfig.EnableSwiftV2 { if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") } // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - // here for AKS(K8s) swiftv2 middleware to process IP configs requests - swiftV2Middleware := &restserver.K8sSWIFTv2Middleware{Cli: manager.GetClient()} + // switch here for different type of swift v2 middleware (k8s or SF) + var swiftV2Middleware cns.IPConfigsHandlerMiddleware + switch cnsconfig.SWIFTV2Mode { + case configuration.K8sSWIFTV2: + swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()} + case configuration.SFSWIFTV2: + default: + // default to K8s middleware for now, in a later changes we where start to pass in + // SWIFT v2 mode in CNS config, this should throw an error if the mode is not set. + swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()} + } httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware) } diff --git a/cns/wireserver/client.go b/cns/wireserver/client.go index 4ae375898c..417e60ef6f 100644 --- a/cns/wireserver/client.go +++ b/cns/wireserver/client.go @@ -11,6 +11,10 @@ import ( "github.com/pkg/errors" ) +const ( + WireserverIP = "168.63.129.16" +) + type GetNetworkContainerOpts struct { NetworkContainerID string PrimaryAddress string diff --git a/cns/wireserver/net.go b/cns/wireserver/net.go index cb7c0cd3eb..ed0791d0af 100644 --- a/cns/wireserver/net.go +++ b/cns/wireserver/net.go @@ -2,16 +2,13 @@ package wireserver import ( "net" - "strings" "github.com/pkg/errors" ) var ( - // ErrNoPrimaryInterface indicates the wireserver response does not have a primary interface indicated. + // ErrNoPrimaryInterface indicates the wireserver respnose does not have a primary interface indicated. ErrNoPrimaryInterface = errors.New("no primary interface found") - // ErrNoSecondaryInterface indicates the wireserver response does not have secondary interface on the node - ErrNoSecondaryInterface = errors.New("no secondary interface found") // ErrInsufficientAddressSpace indicates that the CIDR space is too small to include a gateway IP; it is 1 IP. ErrInsufficientAddressSpace = errors.New("insufficient address space to generate gateway IP") ) @@ -52,48 +49,6 @@ func GetPrimaryInterfaceFromResult(res *GetInterfacesResult) (*InterfaceInfo, er return nil, ErrNoPrimaryInterface } -// Gets secondary interface details for swiftv2 secondary nics scenario -func GetSecondaryInterfaceFromResult(res *GetInterfacesResult, macAddress string) (*InterfaceInfo, error) { - for _, i := range res.Interface { - // skip if primary - if i.IsPrimary { - continue - } - - // skip if no subnets - if len(i.IPSubnet) == 0 { - continue - } - - if macAddressesEqual(i.MacAddress, macAddress) { - // get the second subnet - s := i.IPSubnet[0] - gw, err := calculateGatewayIP(s.Prefix) - if err != nil { - return nil, err - } - - secondaryIP := "" - for _, ip := range s.IPAddress { - if !ip.IsPrimary { - secondaryIP = ip.Address - break - } - } - var secondaryIPs []string - secondaryIPs = append(secondaryIPs, secondaryIP) - - return &InterfaceInfo{ - Subnet: s.Prefix, - IsPrimary: false, - Gateway: gw.String(), - SecondaryIPs: secondaryIPs, - }, nil - } - } - return nil, ErrNoSecondaryInterface -} - // calculateGatewayIP parses the passed CIDR string and returns the first IP in the range. func calculateGatewayIP(cidr string) (net.IP, error) { _, subnet, err := net.ParseCIDR(cidr) @@ -124,10 +79,3 @@ func calculateGatewayIP(cidr string) (net.IP, error) { } return gw, nil } - -func macAddressesEqual(macAddress1, macAddress2 string) bool { - macAddress1 = strings.ToLower(strings.ReplaceAll(macAddress1, ":", "")) - macAddress2 = strings.ToLower(strings.ReplaceAll(macAddress2, ":", "")) - - return macAddress1 == macAddress2 -}