From 3776e2367e14505020cfa1d8d6890a7a22106d42 Mon Sep 17 00:00:00 2001 From: Yuting Peng Date: Mon, 8 Apr 2024 09:45:07 -0700 Subject: [PATCH] extract secondary vnic --- pkg/cloudprovider/providers/oci/instances.go | 21 ++++++++ .../providers/oci/instances_test.go | 18 +++++++ pkg/csi/driver/bv_controller_test.go | 4 ++ pkg/oci/client/compute.go | 48 +++++++++++++++++++ pkg/volume/provisioner/block/block_test.go | 4 ++ pkg/volume/provisioner/fss/fss_test.go | 4 ++ 6 files changed, 99 insertions(+) diff --git a/pkg/cloudprovider/providers/oci/instances.go b/pkg/cloudprovider/providers/oci/instances.go index 986f1457cf..75b1e9c0f6 100644 --- a/pkg/cloudprovider/providers/oci/instances.go +++ b/pkg/cloudprovider/providers/oci/instances.go @@ -97,6 +97,27 @@ func (cp *CloudProvider) extractNodeAddresses(ctx context.Context, instanceID st addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()}) } + secondaryVnic, err := cp.client.Compute().GetSecondaryVNICForInstance(ctx, compartmentID, instanceID) + if err != nil { + return nil, errors.Wrap(err, "GetSecondaryVNICForInstance") + } + + if secondaryVnic.PrivateIp != nil && *secondaryVnic.PrivateIp != "" { + ip := net.ParseIP(*secondaryVnic.PrivateIp) + if ip == nil { + return nil, fmt.Errorf("instance has invalid private address: %q", *secondaryVnic.PrivateIp) + } + addresses = append(addresses, api.NodeAddress{Type: api.NodeInternalIP, Address: ip.String()}) + } + + if secondaryVnic.PublicIp != nil && *secondaryVnic.PublicIp != "" { + ip := net.ParseIP(*secondaryVnic.PublicIp) + if ip == nil { + return nil, errors.Errorf("instance has invalid public address: %q", *secondaryVnic.PublicIp) + } + addresses = append(addresses, api.NodeAddress{Type: api.NodeExternalIP, Address: ip.String()}) + } + // Changing this can have wide reaching impact. // // if vnic.HostnameLabel != nil && *vnic.HostnameLabel != "" { diff --git a/pkg/cloudprovider/providers/oci/instances_test.go b/pkg/cloudprovider/providers/oci/instances_test.go index ccfef626ce..5bbeea9c12 100644 --- a/pkg/cloudprovider/providers/oci/instances_test.go +++ b/pkg/cloudprovider/providers/oci/instances_test.go @@ -340,6 +340,10 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm return instanceVnics[instanceID], nil } +func (MockComputeClient) GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) { + return instanceVnics[instanceID], nil +} + func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil } @@ -762,6 +766,8 @@ func TestExtractNodeAddresses(t *testing.T) { name: "basic-complete", in: "basic-complete", out: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, // v1.NodeAddress{Type: v1.NodeHostName, Address: "basic-complete.subnetwithdnslabel.vcnwithdnslabel.oraclevcn.com"}, @@ -773,6 +779,7 @@ func TestExtractNodeAddresses(t *testing.T) { name: "no-external-ip", in: "no-external-ip", out: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, // v1.NodeAddress{Type: v1.NodeHostName, Address: "no-external-ip.subnetwithdnslabel.vcnwithdnslabel.oraclevcn.com"}, // v1.NodeAddress{Type: v1.NodeInternalDNS, Address: "no-external-ip.subnetwithdnslabel.vcnwithdnslabel.oraclevcn.com"}, @@ -783,6 +790,7 @@ func TestExtractNodeAddresses(t *testing.T) { name: "no-internal-ip", in: "no-internal-ip", out: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, // v1.NodeAddress{Type: v1.NodeHostName, Address: "no-internal-ip.subnetwithdnslabel.vcnwithdnslabel.oraclevcn.com"}, // v1.NodeAddress{Type: v1.NodeInternalDNS, Address: "no-internal-ip.subnetwithdnslabel.vcnwithdnslabel.oraclevcn.com"}, @@ -807,6 +815,8 @@ func TestExtractNodeAddresses(t *testing.T) { out: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, }, err: nil, }, @@ -816,6 +826,8 @@ func TestExtractNodeAddresses(t *testing.T) { out: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, }, err: nil, }, @@ -825,6 +837,8 @@ func TestExtractNodeAddresses(t *testing.T) { out: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, }, err: nil, }, @@ -995,6 +1009,8 @@ func TestNodeAddressesByProviderID(t *testing.T) { out: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, }, err: nil, }, @@ -1004,6 +1020,8 @@ func TestNodeAddressesByProviderID(t *testing.T) { out: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeExternalIP, Address: "0.0.0.1"}, }, err: nil, }, diff --git a/pkg/csi/driver/bv_controller_test.go b/pkg/csi/driver/bv_controller_test.go index 7111ef7e88..35edd35b4d 100644 --- a/pkg/csi/driver/bv_controller_test.go +++ b/pkg/csi/driver/bv_controller_test.go @@ -560,6 +560,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { var page *string var requestMetadata common.RequestMetadata diff --git a/pkg/oci/client/compute.go b/pkg/oci/client/compute.go index e34e311bbc..8f6835de55 100644 --- a/pkg/oci/client/compute.go +++ b/pkg/oci/client/compute.go @@ -33,6 +33,8 @@ type ComputeInterface interface { GetPrimaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) + GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) + VolumeAttachmentInterface } @@ -151,6 +153,52 @@ func (c *client) GetPrimaryVNICForInstance(ctx context.Context, compartmentID, i return nil, errors.WithStack(errNotFound) } +func (c *client) GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) { + logger := c.logger.With("instanceID", instanceID, "compartmentID", compartmentID) + + var page *string + for { + resp, err := c.listVNICAttachments(ctx, core.ListVnicAttachmentsRequest{ + InstanceId: &instanceID, + CompartmentId: &compartmentID, + Page: page, + RequestMetadata: c.requestMetadata, + }) + + if err != nil { + return nil, err + } + + for _, attachment := range resp.Items { + if attachment.LifecycleState != core.VnicAttachmentLifecycleStateAttached { + logger.With("vnicAttachmentID", *attachment.Id).Info("VNIC attachment is not in attached state") + continue + } + + if attachment.VnicId == nil { + // Should never happen but lets be extra cautious as field is non-mandatory in OCI API. + logger.With("vnicAttachmentID", *attachment.Id).Error("VNIC attachment is attached but has no VNIC ID") + continue + } + + // TODO(apryde): Cache map[instanceID]primaryVNICID. + vnic, err := c.GetVNIC(ctx, *attachment.VnicId) + if err != nil { + return nil, err + } + if !*vnic.IsPrimary { + return vnic, nil + } + } + + if page = resp.OpcNextPage; resp.OpcNextPage == nil { + break + } + } + + return nil, errors.WithStack(errNotFound) +} + func (c *client) GetInstanceByNodeName(ctx context.Context, compartmentID, vcnID, nodeName string) (*core.Instance, error) { // First try lookup by display name. instance, err := c.getInstanceByDisplayName(ctx, compartmentID, nodeName) diff --git a/pkg/volume/provisioner/block/block_test.go b/pkg/volume/provisioner/block/block_test.go index 6684f90657..c0ba417ace 100644 --- a/pkg/volume/provisioner/block/block_test.go +++ b/pkg/volume/provisioner/block/block_test.go @@ -241,6 +241,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil } diff --git a/pkg/volume/provisioner/fss/fss_test.go b/pkg/volume/provisioner/fss/fss_test.go index 634f497fe6..b04091e5fe 100644 --- a/pkg/volume/provisioner/fss/fss_test.go +++ b/pkg/volume/provisioner/fss/fss_test.go @@ -240,6 +240,10 @@ func (c *MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compa return nil, nil } +func (c *MockComputeClient) GetSecondaryVNICForInstance(ctx context.Context, compartmentID, instanceID string) (*core.Vnic, error) { + return nil, nil +} + func (c *MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) { return nil, nil }