Skip to content

Commit

Permalink
[NPM Lite] Default Deny CNS Changes (#3286)
Browse files Browse the repository at this point in the history 
* updated CNS for adding default deny acl's

* added infra nic change

* added unit tests

* resolved pr comments

* updating to fix github checks

* added logging lines

* removing unnecessary logging lines

* removed cni circular dependency

* switch from having consts to making them inline

* cns changes based on update to network container contrac

* fixed spelling

* updated unit test

* updated test

* reverted a comment

* updated name of function

* changed policy type

* added a new line

* resolving pr comments

* resolving pr comments

* re-added back

* updated creating acl code to make it more modularized

* fixed golint errors

* fixed golint

* added tests

* fixed spelling

* moved an assertion line

* reformated creating acl's

* refactored code per pr comments

* fixed lint

* moved GetEndpointPolicy so that it is only run on init

* updated code

* updated error message

* updated getEndpointPolicy placement

* updated comment

* fixed golint issues

* refactored

* fixed comments

* updated return inline

* updated unit test returns

* corrected the go lint of file
  • Loading branch information
@rejain456
rejain456 authored Jan 30, 2025
1 parent c670d53 commit e0eb913
Show file tree
Hide file tree
Showing 5 changed files with 397 additions and 116 deletions.
195 changes: 86 additions & 109 deletions cns/middlewares/k8sSwiftV2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,124 +36,33 @@ type K8sSWIFTv2Middleware struct {
// Verify interface compliance at compile time
var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)

// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
// and release IP configs handlers.
func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)

if respCode != types.Success {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: respCode,
Message: message,
},
}, errors.New("failed to validate IP configs request")
}
ipConfigsResp, err := defaultHandler(ctx, req)
// If the pod is not v2, return the response from the handler
if !req.SecondaryInterfacesExist {
return ipConfigsResp, err
}
// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
defer func() {
// Release the default IP config if there is an error
if err != nil {
_, err = failureHandler(ctx, req)
if err != nil {
logger.Errorf("failed to release default IP config : %v", err)
}
}
}()
if err != nil {
return ipConfigsResp, err
}
SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: types.FailedToAllocateIPConfig,
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
},
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
}
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...)
// Set routes for the pod
for i := range ipConfigsResp.PodIPInfo {
ipInfo := &ipConfigsResp.PodIPInfo[i]
// Backend nics doesn't need routes to be set
if ipInfo.NICType != cns.BackendNIC {
err = k.setRoutes(ipInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: types.FailedToAllocateIPConfig,
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
},
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
}
}
}
return ipConfigsResp, nil
func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
// gets pod info for the specified request
podInfo, pod, respCode, message := k.GetPodInfo(ctx, req)
if respCode != types.Success {
return nil, respCode, message
}
}

// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
// nolint
func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
// Retrieve the pod from the cluster
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
if err != nil {
errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
return nil, types.UnexpectedError, errBuf.Error()
}
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
pod := v1.Pod{}
if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
return nil, types.UnexpectedError, errBuf.Error()
}

// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
_, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2]
_, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2]
if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel {
// validates if pod is swiftv2
isSwiftv2 := ValidateSwiftv2Pod(pod)

// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
}
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if !mtpnc.IsReady() {
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
}
// If primary Ip is set in status field, it indicates the presence of secondary interfaces
if mtpnc.Status.PrimaryIP != "" {
req.SecondaryInterfacesExist = true
var mtpnc v1alpha1.MultitenantPodNetworkConfig
// if swiftv2 is enabled, get mtpnc
if isSwiftv2 {
mtpnc, respCode, message = k.getMTPNC(ctx, podInfo)
if respCode != types.Success {
return nil, respCode, message
}
interfaceInfos := mtpnc.Status.InterfaceInfos
for _, interfaceInfo := range interfaceInfos {
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
}
req.BackendInterfaceExist = true
req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)

}
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC {
req.SecondaryInterfacesExist = true
}
// update ipConfigRequest
respCode, message = k.UpdateIPConfigRequest(mtpnc, req)
if respCode != types.Success {
return nil, respCode, message
}
}
logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist)
// retrieve podinfo from orchestrator context

return podInfo, types.Success, ""
}

Expand Down Expand Up @@ -249,3 +158,71 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode {
return cns.K8sSWIFTV2
}

// gets Pod Data
func (k *K8sSWIFTv2Middleware) GetPodInfo(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, k8sPod v1.Pod, respCode types.ResponseCode, message string) {
// Retrieve the pod from the cluster
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
if err != nil {
errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error()
}
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
pod := v1.Pod{}
if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error()
}
return podInfo, pod, types.Success, ""
}

// validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
func ValidateSwiftv2Pod(pod v1.Pod) bool {
// check the pod labels for Swift V2
_, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2]
_, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2]
return swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel
}

func (k *K8sSWIFTv2Middleware) getMTPNC(ctx context.Context, podInfo cns.PodInfo) (mtpncResource v1alpha1.MultitenantPodNetworkConfig, respCode types.ResponseCode, message string) {
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
}
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if !mtpnc.IsReady() {
return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, errMTPNCNotReady.Error()
}
return mtpnc, types.Success, ""
}

// Updates Ip Config Request
func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantPodNetworkConfig, req *cns.IPConfigsRequest) (
respCode types.ResponseCode,
message string,
) {
// If primary Ip is set in status field, it indicates the presence of secondary interfaces
if mtpnc.Status.PrimaryIP != "" {
req.SecondaryInterfacesExist = true
}

interfaceInfos := mtpnc.Status.InterfaceInfos
for _, interfaceInfo := range interfaceInfos {
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
return types.UnexpectedError, errMTPNCNotReady.Error()
}
req.BackendInterfaceExist = true
req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)

}
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC {
req.SecondaryInterfacesExist = true
}
}

return types.Success, ""
}
66 changes: 66 additions & 0 deletions cns/middlewares/k8sSwiftV2_linux.go
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package middlewares

import (
"context"
"fmt"
"net/netip"

"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/configuration"
"github.com/Azure/azure-container-networking/cns/logger"
"github.com/Azure/azure-container-networking/cns/middlewares/utils"
"github.com/Azure/azure-container-networking/cns/types"
"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
"github.com/pkg/errors"
)
Expand Down Expand Up @@ -103,3 +105,67 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo,
}

func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {}

// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
// and release IP configs handlers.
func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
podInfo, respCode, message := k.GetPodInfoForIPConfigsRequest(ctx, &req)

if respCode != types.Success {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: respCode,
Message: message,
},
}, errors.New("failed to validate IP configs request")
}
ipConfigsResp, err := defaultHandler(ctx, req)
// If the pod is not v2, return the response from the handler
if !req.SecondaryInterfacesExist {
return ipConfigsResp, err
}
// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
defer func() {
// Release the default IP config if there is an error
if err != nil {
_, err = failureHandler(ctx, req)
if err != nil {
logger.Errorf("failed to release default IP config : %v", err)
}
}
}()
if err != nil {
return ipConfigsResp, err
}
SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: types.FailedToAllocateIPConfig,
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
},
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
}
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...)
// Set routes for the pod
for i := range ipConfigsResp.PodIPInfo {
ipInfo := &ipConfigsResp.PodIPInfo[i]
// Backend nics doesn't need routes to be set
if ipInfo.NICType != cns.BackendNIC {
err = k.setRoutes(ipInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: types.FailedToAllocateIPConfig,
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
},
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
}
}
}
return ipConfigsResp, nil
}
}
14 changes: 7 additions & 7 deletions cns/middlewares/k8sSwiftV2_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
happyReq.OrchestratorContext = b
happyReq.SecondaryInterfacesExist = false

_, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq)
_, respCode, err := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq)
assert.Equal(t, err, "")
assert.Equal(t, respCode, types.Success)
assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
Expand All @@ -158,7 +158,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
happyReq2.OrchestratorContext = b
happyReq2.SecondaryInterfacesExist = false

_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
_, respCode, err = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq2)
assert.Equal(t, err, "")
assert.Equal(t, respCode, types.Success)
assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
Expand All @@ -172,7 +172,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
happyReq3.OrchestratorContext = b
happyReq3.SecondaryInterfacesExist = false

_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
_, respCode, err = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), happyReq3)
assert.Equal(t, err, "")
assert.Equal(t, respCode, types.Success)
assert.Equal(t, happyReq3.SecondaryInterfacesExist, false)
Expand All @@ -188,7 +188,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
InfraContainerID: testPod1Info.InfraContainerID(),
}
failReq.OrchestratorContext = []byte("invalid")
_, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq)
_, respCode, _ := middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq)
assert.Equal(t, respCode, types.UnexpectedError)

// Pod doesn't exist in cache test
Expand All @@ -198,19 +198,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
}
b, _ := testPod2Info.OrchestratorContext()
failReq.OrchestratorContext = b
_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
_, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq)
assert.Equal(t, respCode, types.UnexpectedError)

// Failed to get MTPNC
b, _ = testPod3Info.OrchestratorContext()
failReq.OrchestratorContext = b
_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
_, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq)
assert.Equal(t, respCode, types.UnexpectedError)

// MTPNC not ready
b, _ = testPod4Info.OrchestratorContext()
failReq.OrchestratorContext = b
_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
_, respCode, _ = middleware.GetPodInfoForIPConfigsRequest(context.TODO(), failReq)
assert.Equal(t, respCode, types.UnexpectedError)
}

Expand Down
Loading

0 comments on commit e0eb913

Please sign in to comment.