Merge pull request #1292 from mesosphere/hectorj2f/kubefed_fed_remote…

…_status

feat: collect remote resource status when enabled

charts/kubefed/README.md

@@ -107,6 +107,7 @@ chart and their default values.
 | controllermanager.commonTolerations   | Tolerations for all the pods.                                                                                                                                                               | []                              |
 | controllermanager.commonNodeSelector     | Node selector for all the pods.                                                                                                                                                          | {}                              |
 | controllermanager.featureGates.PushReconciler               | Push reconciler feature.                                                                                                                                              | true                            |
+| controllermanager.featureGates.RawResourceStatusCollection               | Raw collection of resource status on target clusters feature.                                                                                                                                              | false                            |
 | controllermanager.featureGates.SchedulerPreferences         | Scheduler preferences feature.                                                                                                                                        | true                            |
 | controllermanager.featureGates.CrossClusterServiceDiscovery | Cross cluster service discovery feature.                                                                                                                              | false                           |
 | controllermanager.featureGates.FederatedIngress             | Federated ingress feature.                                                                                                                                            | false                           |

charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml

@@ -24,6 +24,8 @@ spec:
 {{- if .Values.featureGates }}
   - name: PushReconciler
     configuration: {{ .Values.featureGates.PushReconciler | default "Enabled" | quote }}
+  - name: RawResourceStatusCollection
+    configuration: {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }}
   - name: SchedulerPreferences
     configuration: {{ .Values.featureGates.SchedulerPreferences | default "Enabled" | quote }}
   - name: CrossClusterServiceDiscovery

charts/kubefed/crds/crds.yaml

@@ -89,6 +89,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -219,6 +222,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -351,6 +357,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -481,6 +490,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -609,6 +621,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -739,6 +754,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -871,6 +889,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -999,6 +1020,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -1129,6 +1153,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:
@@ -1259,6 +1286,9 @@ spec:
                   properties:
                     name:
                       type: string
+                    remoteStatus:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
                     status:
                       type: string
                   required:

charts/kubefed/values.yaml

@@ -25,6 +25,7 @@ controllermanager:
     SchedulerPreferences:
     CrossClusterServiceDiscovery:
     FederatedIngress:
+    RawResourceStatusCollection:
 
   ## common node selector
   commonNodeSelector: {}

cmd/controller-manager/app/controller-manager.go

@@ -213,6 +213,11 @@ func startControllers(opts *options.Options, stopChan <-chan struct{}) {
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.PushReconciler) {
+		if utilfeature.DefaultFeatureGate.Enabled(features.RawResourceStatusCollection) {
+			opts.Config.RawResourceStatusCollection = true
+			klog.Info("Enabling RawResourceStatusCollection for all the enabled federated resources")
+		}
+
 		if err := federatedtypeconfig.StartController(opts.Config, stopChan); err != nil {
 			klog.Fatalf("Error starting federated type config controller: %v", err)
 		}

config/kubefedconfig.yaml

@@ -16,6 +16,8 @@ spec:
   featureGates:
   - name: PushReconciler
     configuration: "Enabled"
+  - name: RawResourceStatusCollection
+    configuration: "Enabled"
   - name: SchedulerPreferences
     configuration: "Enabled"
   - name: CrossClusterServiceDiscovery

docs/development.md

@@ -98,6 +98,8 @@ types enabled by default:
 1. a label update is reflected in the objects stored in the target
    clusters.
 1. a placement update for the object is reflected in the target clusters.
+1. optionally if `RawResourceStatusCollection` feature is enabled, tests check the value
+of the `remoteStatus` field of federated resources.
 1. deleted resources are removed from the target clusters.
 
 The read operation is implicit.

docs/keps/20200619-federated-resource-status.md

@@ -115,9 +115,9 @@ type GenericFederatedResource struct {
 }
 
 type GenericClusterStatus struct {
-	Name   string            `json:"name"`
-	Status PropagationStatus `json:"status,omitempty"`
-
+  Name   string                  `json:"name"`
+  Status PropagationStatus       `json:"status,omitempty"`
+  RemoteStatus interface{}       `json:"remoteStatus,omitempty"`
   Conditions []*metav1.Condition `json:"conditions,omitempty"`
 }
 
@@ -365,11 +365,6 @@ to expect a common schema for `.status.conditions` and share golang logic for co
 By following this approach, kubefed would be able to properly consume and report the status
 of any federated resource by checking the `status.conditions` (e.g `Ready=True`) fields.
 
-
-**Note that**, this proposed solution might include additional flags to the kubefed control-plane
-components to avoid blowing out the control-plane due to frequent and concurrent API calls to
-update the status of the federated resources.
-
 Obviously users might want to be able to enable/disable the definition of:
 
 * Which condition to look for each federated resource to determine its readiness, e.g. `Ready=True`, `Deployed=True`.
@@ -379,8 +374,99 @@ This might be especially useful for custom resource types without a `IsReady` st
 By raw status, the system understand to show the status of all the federated resources `Ready` and not `Ready`.
 This can have an impact in the performance, so this should be configured with precautions.
 
-To do so, the system exposes properties as part of each `FederatedTypeConfig` to define
-the desired behavior at federated resource type.
+**Note that**, this proposed solution includes additional flags to the kubefed control-plane
+components to limit the calls per second to the API, which could cause performance degradations
+to the control-plane when updating the status of the federated resources.
+
+In addition to the `conditions` field, this new feature adds a new property `remoteStatus` that holds
+the status of the created resource created in the kubefed cluster.The `remoteStatus` is an unstructured
+object due to the unknown structure of the status of the target resources.
+To enable the collection of the status of federated resources, a new feature gate, disabled by default, is added to the current list of features.
+Likewise the user has to explicitly enable the collection at `FederatedTypeConfig` level. To do so, the `FederatedTypeConfig.statusCollection` field
+has to be set to `Enabled`.
+
+As an example, the following property needs to be enabled in the `FederatedDeployment`:
+
+```yaml
+spec:
+  federatedType:
+    group: types.kubefed.io
+    kind: FederatedDeployment
+    pluralName: federateddeployments
+    scope: Namespaced
+    version: v1beta1
+  propagation: Enabled
+  statusCollection: Enabled
+  targetType:
+    group: apps
+    kind: Deployment
+    pluralName: deployments
+    scope: Namespaced
+    version: v1
+```
+
+An example of a `FederatedDeployment` federated in a `cluster1` where the feature
+collected the resource remote status `remoteStatus`.
+
+```yaml
+- apiVersion: types.kubefed.io/v1beta1
+  kind: FederatedDeployment
+  metadata:
+    finalizers:
+    - kubefed.io/sync-controller
+    generation: 1
+    name: asystem
+    namespace: asystem
+    resourceVersion: "70174497"
+  spec:
+    placement:
+      clusters:
+      - name: cluster1
+    template:
+      metadata:
+        labels:
+          app: nginx
+      spec:
+        replicas: 3
+        selector:
+          matchLabels:
+            app: nginx
+        template:
+          metadata:
+            labels:
+              app: nginx
+          spec:
+            containers:
+            - image: nginx
+              name: nginx
+  status:
+    clusters:
+    - name: "cluster1"
+      remoteStatus:
+        availableReplicas: 3
+        conditions:
+        - lastTransitionTime: "2020-09-18T11:07:55Z"
+          lastUpdateTime: "2020-09-18T11:18:31Z"
+          message: ReplicaSet "asystem-f89759699" has successfully progressed.
+          reason: NewReplicaSetAvailable
+          status: "True"
+          type: Progressing
+        - lastTransitionTime: "2020-09-24T05:42:11Z"
+          lastUpdateTime: "2020-09-24T05:42:11Z"
+          message: Deployment has minimum availability.
+          reason: MinimumReplicasAvailable
+          status: "True"
+          type: Available
+        observedGeneration: 3
+        readyReplicas: 3
+        replicas: 3
+        updatedReplicas: 3
+    conditions:
+    - lastTransitionTime: "2020-05-25T20:23:59Z"
+      lastUpdateTime: "2020-05-25T20:23:59Z"
+      status: "True"
+      type: "Propagation"
+```
 
 ### User Stories
 

example/config/kubefedconfig.yaml

@@ -17,6 +17,8 @@ spec:
   featureGates:
   - name: PushReconciler
     configuration: "Enabled"
+  - name: RawResourceStatusCollection
+    configuration: "Enabled"
   - name: SchedulerPreferences
     configuration: "Enabled"
   - name: CrossClusterServiceDiscovery

go.mod

@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/evanphx/json-patch v4.9.0+incompatible
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-logr/logr v0.2.1 // indirect
+	github.com/go-logr/logr v0.3.0 // indirect
 	github.com/go-logr/zapr v0.2.0 // indirect
 	github.com/json-iterator/go v1.1.10
 	github.com/onsi/ginkgo v1.14.2
@@ -16,6 +16,8 @@ require (
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
+	go.uber.org/zap v1.16.0 // indirect
+	golang.org/x/text v0.3.4 // indirect
 	k8s.io/api v0.19.3
 	k8s.io/apiextensions-apiserver v0.19.3
 	k8s.io/apimachinery v0.19.3

go.sum

@@ -177,6 +177,8 @@ github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE=
 github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs=
+github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
 github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
 github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4=
@@ -606,17 +608,23 @@ go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
+go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -755,6 +763,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=

pkg/apis/core/v1beta1/federatedtypeconfig_types.go

@@ -207,18 +207,20 @@ func (f *FederatedTypeConfig) GetFederatedType() metav1.APIResource {
 	return apiResourceToMeta(f.Spec.FederatedType, f.GetFederatedNamespaced())
 }
 
+// TODO (hectorj2f): It should get deprecated once we move to the new status approach
+// because the type is the same as the target type.
 func (f *FederatedTypeConfig) GetStatusType() *metav1.APIResource {
 	if f.Spec.StatusType == nil {
 		return nil
 	}
+	// Return the original target type
 	metaAPIResource := apiResourceToMeta(*f.Spec.StatusType, f.Spec.StatusType.Namespaced())
 	return &metaAPIResource
 }
 
 func (f *FederatedTypeConfig) GetStatusEnabled() bool {
 	return f.Spec.StatusCollection != nil &&
-		*f.Spec.StatusCollection == StatusCollectionEnabled &&
-		f.Name == "services"
+		*f.Spec.StatusCollection == StatusCollectionEnabled
 }
 
 // TODO(font): This method should be removed from the interface i.e. remove

pkg/apis/core/v1beta1/validation/validation.go

@@ -310,7 +310,7 @@ func ValidateKubeFedConfig(kubeFedConfig, oldKubeFedConfig *v1beta1.KubeFedConfi
 			existingNames[gate.Name] = true
 
 			allErrs = append(allErrs, validateEnumStrings(gatesPath.Child("name"), string(gate.Name),
-				[]string{string(features.PushReconciler), string(features.SchedulerPreferences),
+				[]string{string(features.PushReconciler), string(features.RawResourceStatusCollection), string(features.SchedulerPreferences),
 					string(features.CrossClusterServiceDiscovery), string(features.FederatedIngress)})...)
 
 			allErrs = append(allErrs, validateEnumStrings(gatesPath.Child("configuration"), string(gate.Configuration),