From 1655714c9d46ced7cf7905f06d508aa3ec820502 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Date: Mon, 22 Jan 2024 23:22:08 -0800 Subject: [PATCH] GCP NFS Instance Reconcilation --- .../cloud-control/v1beta1/iprange_types.go | 4 + .../v1beta1/nfsinstance_types.go | 4 + ...loud-control.kyma-project.io_ipranges.yaml | 3 + ...-control.kyma-project.io_nfsinstances.yaml | 3 + .../pkg/provider/gcp/client/gcpConstants.go | 4 + .../provider/gcp/iprange/checkGcpOperation.go | 75 +++++++++++++++++++ .../iprange/client/serviceNetworkingClient.go | 4 +- .../kcp/pkg/provider/gcp/iprange/new.go | 1 + .../pkg/provider/gcp/iprange/syncAddress.go | 28 ++++--- .../provider/gcp/iprange/syncPsaConnection.go | 33 ++++---- .../gcp/nfsinstance/checkGcpOperation.go | 48 ++++++++++++ .../gcp/nfsinstance/checkUpdateMask.go | 27 +++++++ .../gcp/nfsinstance/loadNfsInstance.go | 3 +- .../kcp/pkg/provider/gcp/nfsinstance/new.go | 1 + .../kcp/pkg/provider/gcp/nfsinstance/state.go | 1 + .../gcp/nfsinstance/syncNfsInstance.go | 33 ++++---- 16 files changed, 228 insertions(+), 44 deletions(-) create mode 100644 components/kcp/pkg/provider/gcp/iprange/checkGcpOperation.go create mode 100644 components/kcp/pkg/provider/gcp/nfsinstance/checkGcpOperation.go create mode 100644 components/kcp/pkg/provider/gcp/nfsinstance/checkUpdateMask.go diff --git a/components/kcp/api/cloud-control/v1beta1/iprange_types.go b/components/kcp/api/cloud-control/v1beta1/iprange_types.go index 1567e3423..99e5b7cf4 100644 --- a/components/kcp/api/cloud-control/v1beta1/iprange_types.go +++ b/components/kcp/api/cloud-control/v1beta1/iprange_types.go @@ -111,6 +111,10 @@ type IpRangeStatus struct { // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:"conditions,omitempty"` + + // Operation Identifier to track the Hyperscaler Operation + // +optional + OpIdentifier string `json:"opIdentifier,omitempty"` } type IpRangeSubnets []IpRangeSubnet diff --git a/components/kcp/api/cloud-control/v1beta1/nfsinstance_types.go b/components/kcp/api/cloud-control/v1beta1/nfsinstance_types.go index c6cacd0cc..b34fbe4bf 100644 --- a/components/kcp/api/cloud-control/v1beta1/nfsinstance_types.go +++ b/components/kcp/api/cloud-control/v1beta1/nfsinstance_types.go @@ -101,6 +101,10 @@ type NfsInstanceStatus struct { //List of NFS Hosts (DNS Names or IP Addresses) that clients can use to connect // +optional Hosts []string `json:"hosts,omitempty"` + + // Operation Identifier to track the Hyperscaler Operation + // +optional + OpIdentifier string `json:"opIdentifier,omitempty"` } //+kubebuilder:object:root=true diff --git a/components/kcp/config/crd/bases/cloud-control.kyma-project.io_ipranges.yaml b/components/kcp/config/crd/bases/cloud-control.kyma-project.io_ipranges.yaml index e88363d21..f6f5102fd 100644 --- a/components/kcp/config/crd/bases/cloud-control.kyma-project.io_ipranges.yaml +++ b/components/kcp/config/crd/bases/cloud-control.kyma-project.io_ipranges.yaml @@ -164,6 +164,9 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + opIdentifier: + description: Operation Identifier to track the Hyperscaler Operation + type: string ranges: items: type: string diff --git a/components/kcp/config/crd/bases/cloud-control.kyma-project.io_nfsinstances.yaml b/components/kcp/config/crd/bases/cloud-control.kyma-project.io_nfsinstances.yaml index d19c36ad3..9ef4e8ef1 100644 --- a/components/kcp/config/crd/bases/cloud-control.kyma-project.io_nfsinstances.yaml +++ b/components/kcp/config/crd/bases/cloud-control.kyma-project.io_nfsinstances.yaml @@ -201,6 +201,9 @@ spec: type: array id: type: string + opIdentifier: + description: Operation Identifier to track the Hyperscaler Operation + type: string state: type: string type: object diff --git a/components/kcp/pkg/provider/gcp/client/gcpConstants.go b/components/kcp/pkg/provider/gcp/client/gcpConstants.go index d24086499..f99597053 100644 --- a/components/kcp/pkg/provider/gcp/client/gcpConstants.go +++ b/components/kcp/pkg/provider/gcp/client/gcpConstants.go @@ -2,6 +2,7 @@ package client import ( "fmt" + "time" "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" ) @@ -14,6 +15,9 @@ const PsaPeeringName = "servicenetworking-googleapis-com" const filestoreInstancePattern = "projects/%s/locations/%s/instances/%s" const filestoreParentPattern = "projects/%s/locations/%s" +const GcpRetryWaitTime = time.Second * 3 +const GcpOperationWaitTime = time.Second * 5 + func GetVPCPath(projectId, vpcId string) string { return fmt.Sprintf(vPCPathPattern, projectId, vpcId) } diff --git a/components/kcp/pkg/provider/gcp/iprange/checkGcpOperation.go b/components/kcp/pkg/provider/gcp/iprange/checkGcpOperation.go new file mode 100644 index 000000000..bd8385b02 --- /dev/null +++ b/components/kcp/pkg/provider/gcp/iprange/checkGcpOperation.go @@ -0,0 +1,75 @@ +package iprange + +import ( + "context" + "errors" + + "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" + "github.com/kyma-project/cloud-manager/components/lib/composed" +) + +func checkGcpOperation(ctx context.Context, st composed.State) (error, context.Context) { + state := st.(*State) + logger := composed.LoggerFromCtx(ctx) + + ipRange := state.ObjAsIpRange() + opName := ipRange.Status.OpIdentifier + logger.WithValues("ipRange :", ipRange.Name).Info("Checking Operation") + + //If no OpIdentifier, then continue to next action. + if opName == "" { + return nil, nil + } + + //Check SyncPsa Operation.. + if ipRange.Status.State == client.SyncPsaConnection || + ipRange.Status.State == client.DeletePsaConnection { + op, err := state.serviceNetworkingClient.GetOperation(ctx, opName) + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil) + } + + //Operation not completed yet.. requeue again. + if op != nil && !op.Done { + return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil + } + + //If not able to find the operation or it is completed, reset OpIdentifier. + if op == nil || op.Done { + ipRange.Status.OpIdentifier = "" + } + + //If the operation failed, update the error status on the object. + if op != nil && op.Error != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.Error.Message)) + } + } else if ipRange.Status.State == client.SyncAddress || + ipRange.Status.State == client.DeleteAddress { + project := state.Scope().Spec.Scope.Gcp.Project + op, err := state.computeClient.GetGlobalOperation(ctx, project, opName) + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil) + } + + //Operation not completed yet.. requeue again. + if op != nil && op.Status != "DONE" { + return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil + } + + //If not able to find the operation or it is completed, reset OpIdentifier. + if op == nil || op.Status == "DONE" { + ipRange.Status.OpIdentifier = "" + } + + //If the operation failed, update the error status on the object. + if op != nil && op.Error != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.StatusMessage)) + } + + } + + return nil, nil +} diff --git a/components/kcp/pkg/provider/gcp/iprange/client/serviceNetworkingClient.go b/components/kcp/pkg/provider/gcp/iprange/client/serviceNetworkingClient.go index 595c54255..ff2989cea 100644 --- a/components/kcp/pkg/provider/gcp/iprange/client/serviceNetworkingClient.go +++ b/components/kcp/pkg/provider/gcp/iprange/client/serviceNetworkingClient.go @@ -2,10 +2,11 @@ package client import ( "context" - "github.com/kyma-project/cloud-manager/components/lib/composed" "net/http" "strconv" + "github.com/kyma-project/cloud-manager/components/lib/composed" + gcpclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" "google.golang.org/api/option" "google.golang.org/api/servicenetworking/v1" @@ -18,6 +19,7 @@ type ServiceNetworkingClient interface { // projectNumber: Project number which is different from project id. Get it by calling client.GetProjectNumber(ctx, projectId) DeleteServiceConnection(ctx context.Context, projectNumber int64, vpcId string) (*servicenetworking.Operation, error) PatchServiceConnection(ctx context.Context, projectId, vpcId string, reservedIpRanges []string) (*servicenetworking.Operation, error) + GetOperation(ctx context.Context, operationName string) (*servicenetworking.Operation, error) } func NewServiceNetworkingClient() gcpclient.ClientProvider[ServiceNetworkingClient] { diff --git a/components/kcp/pkg/provider/gcp/iprange/new.go b/components/kcp/pkg/provider/gcp/iprange/new.go index a0d1c3bda..3bbdc67f3 100644 --- a/components/kcp/pkg/provider/gcp/iprange/new.go +++ b/components/kcp/pkg/provider/gcp/iprange/new.go @@ -25,6 +25,7 @@ func New(stateFactory StateFactory) composed.Action { "gcpIpRange", validateCidr, focal.AddFinalizer, + checkGcpOperation, loadAddress, loadPsaConnection, compareStates, diff --git a/components/kcp/pkg/provider/gcp/iprange/syncAddress.go b/components/kcp/pkg/provider/gcp/iprange/syncAddress.go index 0be026e4f..6aaaca192 100644 --- a/components/kcp/pkg/provider/gcp/iprange/syncAddress.go +++ b/components/kcp/pkg/provider/gcp/iprange/syncAddress.go @@ -6,7 +6,9 @@ import ( "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" "github.com/kyma-project/cloud-manager/components/lib/composed" + "google.golang.org/api/compute/v1" ) func syncAddress(ctx context.Context, st composed.State) (error, context.Context) { @@ -20,24 +22,26 @@ func syncAddress(ctx context.Context, st composed.State) (error, context.Context project := gcpScope.Project vpc := gcpScope.VpcNetwork + var operation *compute.Operation + var err error switch state.addressOp { case focal.ADD: - _, err := state.computeClient.CreatePscIpRange(ctx, project, vpc, ipRange.Name, ipRange.Name, state.ipAddress, int64(state.prefix)) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error creating Address object in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.computeClient.CreatePscIpRange(ctx, project, vpc, ipRange.Name, ipRange.Name, state.ipAddress, int64(state.prefix)) case focal.MODIFY: - err := errors.New("IpRange update not supported.") + err = errors.New("IpRange update not supported.") state.AddErrorCondition(ctx, v1beta1.ReasonNotSupported, err) return composed.LogErrorAndReturn(err, "IpRange update not supported.", composed.StopAndForget, nil) case focal.DELETE: - _, err := state.computeClient.DeleteIpRange(ctx, project, ipRange.Name) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error deleting address object in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.computeClient.DeleteIpRange(ctx, project, ipRange.Name) } - return composed.StopWithRequeue, nil + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error synchronizing Address object in GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil) + } + if operation != nil { + ipRange.Status.OpIdentifier = operation.Name + state.UpdateObjStatus(ctx) + } + return composed.StopWithRequeueDelay(client.GcpOperationWaitTime), nil } diff --git a/components/kcp/pkg/provider/gcp/iprange/syncPsaConnection.go b/components/kcp/pkg/provider/gcp/iprange/syncPsaConnection.go index 159a477cd..772caf247 100644 --- a/components/kcp/pkg/provider/gcp/iprange/syncPsaConnection.go +++ b/components/kcp/pkg/provider/gcp/iprange/syncPsaConnection.go @@ -5,7 +5,9 @@ import ( "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" "github.com/kyma-project/cloud-manager/components/lib/composed" + "google.golang.org/api/servicenetworking/v1" ) func syncPsaConnection(ctx context.Context, st composed.State) (error, context.Context) { @@ -19,26 +21,25 @@ func syncPsaConnection(ctx context.Context, st composed.State) (error, context.C project := gcpScope.Project vpc := gcpScope.VpcNetwork + var operation *servicenetworking.Operation + var err error + switch state.connectionOp { case focal.ADD: - _, err := state.serviceNetworkingClient.CreateServiceConnection(ctx, project, vpc, state.ipRanges) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error creating Service Connections in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.serviceNetworkingClient.CreateServiceConnection(ctx, project, vpc, state.ipRanges) case focal.MODIFY: - _, err := state.serviceNetworkingClient.PatchServiceConnection(ctx, project, vpc, state.ipRanges) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error patching Service Connections in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.serviceNetworkingClient.PatchServiceConnection(ctx, project, vpc, state.ipRanges) case focal.DELETE: - _, err := state.serviceNetworkingClient.DeleteServiceConnection(ctx, state.projectNumber, vpc) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error deleting Service Connections in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.serviceNetworkingClient.DeleteServiceConnection(ctx, state.projectNumber, vpc) } - return composed.StopWithRequeue, nil + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error syncronizing Service Connections in GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil) + } + if operation != nil { + ipRange.Status.OpIdentifier = operation.Name + state.UpdateObjStatus(ctx) + } + return composed.StopWithRequeueDelay(client.GcpOperationWaitTime), nil } diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/checkGcpOperation.go b/components/kcp/pkg/provider/gcp/nfsinstance/checkGcpOperation.go new file mode 100644 index 000000000..11eee49a8 --- /dev/null +++ b/components/kcp/pkg/provider/gcp/nfsinstance/checkGcpOperation.go @@ -0,0 +1,48 @@ +package nfsinstance + +import ( + "context" + "errors" + + "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" + "github.com/kyma-project/cloud-manager/components/lib/composed" +) + +func checkGcpOperation(ctx context.Context, st composed.State) (error, context.Context) { + state := st.(*State) + logger := composed.LoggerFromCtx(ctx) + + nfsInstance := state.ObjAsNfsInstance() + opName := nfsInstance.Status.OpIdentifier + logger.WithValues("NfsInstance :", nfsInstance.Name).Info("Checking GCP Operation Status") + + //If no OpIdentifier, then continue to next action. + if opName == "" { + return nil, nil + } + + project := state.Scope().Spec.Scope.Gcp.Project + op, err := state.filestoreClient.GetOperation(ctx, project, opName) + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error getting operation from GCP", composed.StopWithRequeue, nil) + } + + //Operation not completed yet.. requeue again. + if op != nil && !op.Done { + return composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil + } + + //If not able to find the operation or it is completed, reset OpIdentifier. + if op == nil || op.Done { + nfsInstance.Status.OpIdentifier = "" + } + + //If the operation failed, update the error status on the object. + if op != nil && op.Error != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, errors.New(op.Error.Message)) + } + + return nil, nil +} diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/checkUpdateMask.go b/components/kcp/pkg/provider/gcp/nfsinstance/checkUpdateMask.go new file mode 100644 index 000000000..528fd1c58 --- /dev/null +++ b/components/kcp/pkg/provider/gcp/nfsinstance/checkUpdateMask.go @@ -0,0 +1,27 @@ +package nfsinstance + +import ( + "context" + + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" + "github.com/kyma-project/cloud-manager/components/lib/composed" +) + +func checkUpdateMask(ctx context.Context, st composed.State) (error, context.Context) { + state := st.(*State) + logger := composed.LoggerFromCtx(ctx) + + nfsInstance := state.ObjAsNfsInstance() + logger.WithValues("NfsInstance :", nfsInstance.Name).Info("Checking for Update Mask") + + //If the operation is not modify, continue. + if state.operation != focal.MODIFY { + return nil, nil + } + + //If capacity is increased, add it to updateMask + if nfsInstance.Spec.Instance.Gcp.CapacityGb > int(state.fsInstance.FileShares[0].CapacityGb) { + state.updateMask = append(state.updateMask, "FileShares") + } + return nil, nil +} diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/loadNfsInstance.go b/components/kcp/pkg/provider/gcp/nfsinstance/loadNfsInstance.go index a15cde09a..6c66dcd15 100644 --- a/components/kcp/pkg/provider/gcp/nfsinstance/loadNfsInstance.go +++ b/components/kcp/pkg/provider/gcp/nfsinstance/loadNfsInstance.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" "github.com/kyma-project/cloud-manager/components/lib/composed" "google.golang.org/api/googleapi" ) @@ -33,7 +34,7 @@ func loadNfsInstance(ctx context.Context, st composed.State) (error, context.Con } } state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error getting Filestore Instance from GCP", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error getting Filestore Instance from GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil) } //Store the fsInstance in state diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/new.go b/components/kcp/pkg/provider/gcp/nfsinstance/new.go index 4790466de..4211be501 100644 --- a/components/kcp/pkg/provider/gcp/nfsinstance/new.go +++ b/components/kcp/pkg/provider/gcp/nfsinstance/new.go @@ -24,6 +24,7 @@ func New(stateFactory StateFactory) composed.Action { return composed.ComposeActions( "gcsNfsInstance", focal.AddFinalizer, + checkGcpOperation, loadNfsInstance, checkNUpdateState, syncNfsInstance, diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/state.go b/components/kcp/pkg/provider/gcp/nfsinstance/state.go index 527128047..d99bc6e13 100644 --- a/components/kcp/pkg/provider/gcp/nfsinstance/state.go +++ b/components/kcp/pkg/provider/gcp/nfsinstance/state.go @@ -16,6 +16,7 @@ type State struct { types.State curState v1beta1.StatusState operation focal.OperationType + updateMask []string fsInstance *file.Instance filestoreClient client.FilestoreClient } diff --git a/components/kcp/pkg/provider/gcp/nfsinstance/syncNfsInstance.go b/components/kcp/pkg/provider/gcp/nfsinstance/syncNfsInstance.go index fdc2020f9..b142783a2 100644 --- a/components/kcp/pkg/provider/gcp/nfsinstance/syncNfsInstance.go +++ b/components/kcp/pkg/provider/gcp/nfsinstance/syncNfsInstance.go @@ -2,11 +2,13 @@ package nfsinstance import ( "context" - "errors" + "strings" "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" "github.com/kyma-project/cloud-manager/components/lib/composed" + "google.golang.org/api/file/v1" ) func syncNfsInstance(ctx context.Context, st composed.State) (error, context.Context) { @@ -22,24 +24,27 @@ func syncNfsInstance(ctx context.Context, st composed.State) (error, context.Con location := state.getGcpLocation() name := nfsInstance.Spec.RemoteRef.Name + var operation *file.Operation + var err error + switch state.operation { case focal.ADD: - _, err := state.filestoreClient.CreateFilestoreInstance(ctx, project, location, name, state.toInstance()) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error creating Filestore object in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.filestoreClient.CreateFilestoreInstance(ctx, project, location, name, state.toInstance()) case focal.MODIFY: - err := errors.New("Filestore update not supported.") - state.AddErrorCondition(ctx, v1beta1.ReasonNotSupported, err) - return composed.LogErrorAndReturn(err, "Filestore update not supported.", composed.StopAndForget, nil) + mask := strings.Join(state.updateMask, ",") + operation, err = state.filestoreClient.PatchFilestoreInstance(ctx, project, location, name, mask, state.toInstance()) case focal.DELETE: - _, err := state.filestoreClient.DeleteFilestoreInstance(ctx, project, location, name) - if err != nil { - state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) - return composed.LogErrorAndReturn(err, "Error deleting Filestore object in GCP", composed.StopWithRequeue, nil) - } + operation, err = state.filestoreClient.DeleteFilestoreInstance(ctx, project, location, name) + } + + if err != nil { + state.AddErrorCondition(ctx, v1beta1.ReasonGcpError, err) + return composed.LogErrorAndReturn(err, "Error synchronizing Filestore object in GCP", composed.StopWithRequeueDelay(client.GcpRetryWaitTime), nil) } + if operation != nil { + nfsInstance.Status.OpIdentifier = operation.Name + state.UpdateObjStatus(ctx) + } return nil, nil }