GCP NFS Instance Reconcilation

components/kcp/api/cloud-control/v1beta1/iprange_types.go

@@ -111,6 +111,10 @@ type IpRangeStatus struct {
 	// +listType=map
 	// +listMapKey=type
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// Operation Identifier to track the Hyperscaler Operation
+	// +optional
+	OpIdentifier string `json:"opIdentifier,omitempty"`
 }
 
 type IpRangeSubnets []IpRangeSubnet

components/kcp/api/cloud-control/v1beta1/nfsinstance_types.go

@@ -101,6 +101,10 @@ type NfsInstanceStatus struct {
 	//List of NFS Hosts (DNS Names or IP Addresses) that clients can use to connect
 	// +optional
 	Hosts []string `json:"hosts,omitempty"`
+
+	// Operation Identifier to track the Hyperscaler Operation
+	// +optional
+	OpIdentifier string `json:"opIdentifier,omitempty"`
 }
 
 //+kubebuilder:object:root=true

components/kcp/config/crd/bases/cloud-control.kyma-project.io_ipranges.yaml

@@ -164,6 +164,9 @@ spec:
                 x-kubernetes-list-map-keys:
                 - type
                 x-kubernetes-list-type: map
+              opIdentifier:
+                description: Operation Identifier to track the Hyperscaler Operation
+                type: string
               ranges:
                 items:
                   type: string

components/kcp/config/crd/bases/cloud-control.kyma-project.io_nfsinstances.yaml

@@ -201,6 +201,9 @@ spec:
                 type: array
               id:
                 type: string
+              opIdentifier:
+                description: Operation Identifier to track the Hyperscaler Operation
+                type: string
               state:
                 type: string
             type: object

components/kcp/pkg/provider/gcp/client/gcpConstants.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
 )
@@ -14,6 +15,9 @@ const PsaPeeringName = "servicenetworking-googleapis-com"
 const filestoreInstancePattern = "projects/%s/locations/%s/instances/%s"
 const filestoreParentPattern = "projects/%s/locations/%s"
 
+const GcpRetryWaitTime = time.Second * 3
+const GcpOperationWaitTime = time.Second * 5
+
 func GetVPCPath(projectId, vpcId string) string {
 	return fmt.Sprintf(vPCPathPattern, projectId, vpcId)
 }

components/kcp/pkg/provider/gcp/iprange/checkGcpOperation.go

@@ -0,0 +1,75 @@
+package iprange
+
+import (
+	"context"
+	"errors"
+
+	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
+	"github.com/kyma-project/cloud-manager/components/lib/composed"
+)
+
+func checkGcpOperation(ctx context.Context, st composed.State) (error, context.Context) {
+	state := st.(*State)
+	logger := composed.LoggerFromCtx(ctx)
+
+	ipRange := state.ObjAsIpRange()
+	opName := ipRange.Status.OpIdentifier
+	logger.WithValues("ipRange :", ipRange.Name).Info("Checking Operation")
+
+	//If no OpIdentifier, then continue to next action.
+	if opName == "" {
+		return nil, nil
+	}
+
+	//Check SyncPsa Operation..
+	if ipRange.Status.State == client.SyncPsaConnection ||
+		ipRange.Status.State == client.DeletePsaConnection {
+		op, err := state.serviceNetworkingClient.GetOperation(ctx, opName)
+		if err != nil {
+			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
+			return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil)
+		}
+
+		//Operation not completed yet.. requeue again.
+		if op != nil && !op.Done {
+			return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil
+		}
+
+		//If not able to find the operation or it is completed, reset OpIdentifier.
+		if op == nil || op.Done {
+			ipRange.Status.OpIdentifier = ""
+		}
+
+		//If the operation failed, update the error status on the object.
+		if op != nil && op.Error != nil {
+			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.Error.Message))
+		}
+	} else if ipRange.Status.State == client.SyncAddress ||
+		ipRange.Status.State == client.DeleteAddress {
+		project := state.Scope().Spec.Scope.Gcp.Project
+		op, err := state.computeClient.GetGlobalOperation(ctx, project, opName)
+		if err != nil {
+			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
+			return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil)
+		}
+
+		//Operation not completed yet.. requeue again.
+		if op != nil && op.Status != "DONE" {
+			return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil
+		}
+
+		//If not able to find the operation or it is completed, reset OpIdentifier.
+		if op == nil || op.Status == "DONE" {
+			ipRange.Status.OpIdentifier = ""
+		}
+
+		//If the operation failed, update the error status on the object.
+		if op != nil && op.Error != nil {
+			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.StatusMessage))
+		}
+
+	}
+
+	return nil, nil
+}

components/kcp/pkg/provider/gcp/iprange/client/serviceNetworkingClient.go

@@ -2,10 +2,11 @@ package client
 
 import (
 	"context"
-	"github.com/kyma-project/cloud-manager/components/lib/composed"
 	"net/http"
 	"strconv"
 
+	"github.com/kyma-project/cloud-manager/components/lib/composed"
+
 	gcpclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
 	"google.golang.org/api/option"
 	"google.golang.org/api/servicenetworking/v1"
@@ -18,6 +19,7 @@ type ServiceNetworkingClient interface {
 	// projectNumber: Project number which is different from project id. Get it by calling client.GetProjectNumber(ctx, projectId)
 	DeleteServiceConnection(ctx context.Context, projectNumber int64, vpcId string) (*servicenetworking.Operation, error)
 	PatchServiceConnection(ctx context.Context, projectId, vpcId string, reservedIpRanges []string) (*servicenetworking.Operation, error)
+	GetOperation(ctx context.Context, operationName string) (*servicenetworking.Operation, error)
 }
 
 func NewServiceNetworkingClient() gcpclient.ClientProvider[ServiceNetworkingClient] {

components/kcp/pkg/provider/gcp/iprange/new.go

@@ -25,6 +25,7 @@ func New(stateFactory StateFactory) composed.Action {
 			"gcpIpRange",
 			validateCidr,
 			focal.AddFinalizer,
+			checkGcpOperation,
 			loadAddress,
 			loadPsaConnection,
 			compareStates,

components/kcp/pkg/provider/gcp/iprange/syncAddress.go

@@ -6,7 +6,9 @@ import (
 
 	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
 	"github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal"
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
 	"github.com/kyma-project/cloud-manager/components/lib/composed"
+	"google.golang.org/api/compute/v1"
 )
 
 func syncAddress(ctx context.Context, st composed.State) (error, context.Context) {
@@ -20,24 +22,26 @@ func syncAddress(ctx context.Context, st composed.State) (error, context.Context
 	project := gcpScope.Project
 	vpc := gcpScope.VpcNetwork
 
+	var operation *compute.Operation
+	var err error
 	switch state.addressOp {
 	case focal.ADD:
-		_, err := state.computeClient.CreatePscIpRange(ctx, project, vpc, ipRange.Name, ipRange.Name, state.ipAddress, int64(state.prefix))
-		if err != nil {
-			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-			return composed.LogErrorAndReturn(err, "Error creating Address object in GCP", composed.StopWithRequeue, nil)
-		}
+		operation, err = state.computeClient.CreatePscIpRange(ctx, project, vpc, ipRange.Name, ipRange.Name, state.ipAddress, int64(state.prefix))
 	case focal.MODIFY:
-		err := errors.New("IpRange update not supported.")
+		err = errors.New("IpRange update not supported.")
 		state.AddErrorCondition(ctx, v1beta1.ReasonNotSupported, err)
 		return composed.LogErrorAndReturn(err, "IpRange update not supported.", composed.StopAndForget, nil)
 	case focal.DELETE:
-		_, err := state.computeClient.DeleteIpRange(ctx, project, ipRange.Name)
-		if err != nil {
-			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-			return composed.LogErrorAndReturn(err, "Error deleting address object in GCP", composed.StopWithRequeue, nil)
-		}
+		operation, err = state.computeClient.DeleteIpRange(ctx, project, ipRange.Name)
 	}
 
-	return composed.StopWithRequeue, nil
+	if err != nil {
+		state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
+		return composed.LogErrorAndReturn(err, "Error synchronizing Address object in GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil)
+	}
+	if operation != nil {
+		ipRange.Status.OpIdentifier = operation.Name
+		state.UpdateObjStatus(ctx)
+	}
+	return composed.StopWithRequeueDelay(client.GcpOperationWaitTime), nil
 }

components/kcp/pkg/provider/gcp/iprange/syncPsaConnection.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
 	"github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal"
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
 	"github.com/kyma-project/cloud-manager/components/lib/composed"
+	"google.golang.org/api/servicenetworking/v1"
 )
 
 func syncPsaConnection(ctx context.Context, st composed.State) (error, context.Context) {
@@ -19,26 +21,25 @@ func syncPsaConnection(ctx context.Context, st composed.State) (error, context.C
 	project := gcpScope.Project
 	vpc := gcpScope.VpcNetwork
 
+	var operation *servicenetworking.Operation
+	var err error
+
 	switch state.connectionOp {
 	case focal.ADD:
-		_, err := state.serviceNetworkingClient.CreateServiceConnection(ctx, project, vpc, state.ipRanges)
-		if err != nil {
-			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-			return composed.LogErrorAndReturn(err, "Error creating Service Connections in GCP", composed.StopWithRequeue, nil)
-		}
+		operation, err = state.serviceNetworkingClient.CreateServiceConnection(ctx, project, vpc, state.ipRanges)
 	case focal.MODIFY:
-		_, err := state.serviceNetworkingClient.PatchServiceConnection(ctx, project, vpc, state.ipRanges)
-		if err != nil {
-			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-			return composed.LogErrorAndReturn(err, "Error patching Service Connections in GCP", composed.StopWithRequeue, nil)
-		}
+		operation, err = state.serviceNetworkingClient.PatchServiceConnection(ctx, project, vpc, state.ipRanges)
 	case focal.DELETE:
-		_, err := state.serviceNetworkingClient.DeleteServiceConnection(ctx, state.projectNumber, vpc)
-		if err != nil {
-			state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-			return composed.LogErrorAndReturn(err, "Error deleting Service Connections in GCP", composed.StopWithRequeue, nil)
-		}
+		operation, err = state.serviceNetworkingClient.DeleteServiceConnection(ctx, state.projectNumber, vpc)
 	}
 
-	return composed.StopWithRequeue, nil
+	if err != nil {
+		state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
+		return composed.LogErrorAndReturn(err, "Error syncronizing Service Connections in GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil)
+	}
+	if operation != nil {
+		ipRange.Status.OpIdentifier = operation.Name
+		state.UpdateObjStatus(ctx)
+	}
+	return composed.StopWithRequeueDelay(client.GcpOperationWaitTime), nil
 }

components/kcp/pkg/provider/gcp/nfsinstance/checkGcpOperation.go

@@ -0,0 +1,48 @@
+package nfsinstance
+
+import (
+	"context"
+	"errors"
+
+	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
+	"github.com/kyma-project/cloud-manager/components/lib/composed"
+)
+
+func checkGcpOperation(ctx context.Context, st composed.State) (error, context.Context) {
+	state := st.(*State)
+	logger := composed.LoggerFromCtx(ctx)
+
+	nfsInstance := state.ObjAsNfsInstance()
+	opName := nfsInstance.Status.OpIdentifier
+	logger.WithValues("NfsInstance :", nfsInstance.Name).Info("Checking GCP Operation Status")
+
+	//If no OpIdentifier, then continue to next action.
+	if opName == "" {
+		return nil, nil
+	}
+
+	project := state.Scope().Spec.Scope.Gcp.Project
+	op, err := state.filestoreClient.GetOperation(ctx, project, opName)
+	if err != nil {
+		state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
+		return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil)
+	}
+
+	//Operation not completed yet.. requeue again.
+	if op != nil && !op.Done {
+		return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil
+	}
+
+	//If not able to find the operation or it is completed, reset OpIdentifier.
+	if op == nil || op.Done {
+		nfsInstance.Status.OpIdentifier = ""
+	}
+
+	//If the operation failed, update the error status on the object.
+	if op != nil && op.Error != nil {
+		state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.Error.Message))
+	}
+
+	return nil, nil
+}

components/kcp/pkg/provider/gcp/nfsinstance/checkUpdateMask.go

@@ -0,0 +1,27 @@
+package nfsinstance
+
+import (
+	"context"
+
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal"
+	"github.com/kyma-project/cloud-manager/components/lib/composed"
+)
+
+func checkUpdateMask(ctx context.Context, st composed.State) (error, context.Context) {
+	state := st.(*State)
+	logger := composed.LoggerFromCtx(ctx)
+
+	nfsInstance := state.ObjAsNfsInstance()
+	logger.WithValues("NfsInstance :", nfsInstance.Name).Info("Checking for Update Mask")
+
+	//If the operation is not modify, continue.
+	if state.operation != focal.MODIFY {
+		return nil, nil
+	}
+
+	//If capacity is increased, add it to updateMask
+	if nfsInstance.Spec.Instance.Gcp.CapacityGb > int(state.fsInstance.FileShares[0].CapacityGb) {
+		state.updateMask = append(state.updateMask, "FileShares")
+	}
+	return nil, nil
+}

components/kcp/pkg/provider/gcp/nfsinstance/loadNfsInstance.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1"
+	"github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client"
 	"github.com/kyma-project/cloud-manager/components/lib/composed"
 	"google.golang.org/api/googleapi"
 )
@@ -33,7 +34,7 @@ func loadNfsInstance(ctx context.Context, st composed.State) (error, context.Con
 			}
 		}
 		state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err)
-		return composed.LogErrorAndReturn(err, "Error getting Filestore Instance from GCP", composed.StopWithRequeue, nil)
+		return composed.LogErrorAndReturn(err, "Error getting Filestore Instance from GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil)
 	}
 
 	//Store the fsInstance in state

components/kcp/pkg/provider/gcp/nfsinstance/new.go

@@ -24,6 +24,7 @@ func New(stateFactory StateFactory) composed.Action {
 		return composed.ComposeActions(
 			"gcsNfsInstance",
 			focal.AddFinalizer,
+			checkGcpOperation,
 			loadNfsInstance,
 			checkNUpdateState,
 			syncNfsInstance,

components/kcp/pkg/provider/gcp/nfsinstance/state.go

@@ -16,6 +16,7 @@ type State struct {
 	types.State
 	curState        v1beta1.StatusState
 	operation       focal.OperationType
+	updateMask      []string
 	fsInstance      *file.Instance
 	filestoreClient client.FilestoreClient
 }