diff --git a/operator/api/v1beta1/aistore_types.go b/operator/api/v1beta1/aistore_types.go index 6d3b7879..cb415167 100644 --- a/operator/api/v1beta1/aistore_types.go +++ b/operator/api/v1beta1/aistore_types.go @@ -77,6 +77,14 @@ type AIStoreSpec struct { // +optional ClusterDomain *string `json:"clusterDomain,omitempty"` + // Secret name containing GCP credentials + // +optional + GCPSecretName *string `json:"gcpSecretName,omitempty"` + + // Secret name containing AWS credentials + // +optional + AWSSecretName *string `json:"awsSecretName,omitempty"` + // ImagePullScerets is an optional list of references to secrets in the same namespace to pull container images of AIS Daemons // More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod // +optional diff --git a/operator/api/v1beta1/zz_generated.deepcopy.go b/operator/api/v1beta1/zz_generated.deepcopy.go index 81ee4d68..dabec63a 100644 --- a/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/operator/api/v1beta1/zz_generated.deepcopy.go @@ -98,6 +98,16 @@ func (in *AIStoreSpec) DeepCopyInto(out *AIStoreSpec) { *out = new(string) **out = **in } + if in.GCPSecretName != nil { + in, out := &in.GCPSecretName, &out.GCPSecretName + *out = new(string) + **out = **in + } + if in.AWSSecretName != nil { + in, out := &in.AWSSecretName, &out.AWSSecretName + *out = new(string) + **out = **in + } if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) diff --git a/operator/config/crd/bases/ais.nvidia.com_aistores.yaml b/operator/config/crd/bases/ais.nvidia.com_aistores.yaml index c514fec1..0ed7a13a 100644 --- a/operator/config/crd/bases/ais.nvidia.com_aistores.yaml +++ b/operator/config/crd/bases/ais.nvidia.com_aistores.yaml @@ -36,6 +36,9 @@ spec: spec: description: AIStoreSpec defines the desired state of AIStore properties: + awsSecretName: + description: Secret name containing AWS credentials + type: string cleanupData: description: Defines if the PVCs and (meta)data should be cleaned up when cluster is destroyed. Reclaiming of PVs associated with @@ -331,6 +334,9 @@ spec: enablePromExporter: description: Defines if AIS daemons should expose prometheus metrics type: boolean + gcpSecretName: + description: Secret name containing GCP credentials + type: string hostpathPrefix: type: string imagePullSecrets: diff --git a/operator/pkg/resources/cmn/configmap.go b/operator/pkg/resources/cmn/configmap.go index 82517852..d1e458ea 100644 --- a/operator/pkg/resources/cmn/configmap.go +++ b/operator/pkg/resources/cmn/configmap.go @@ -32,6 +32,17 @@ func NewGlobalCM(ais *aisv1.AIStore, toUpdate *aiscmn.ConfigToUpdate) (*corev1.C return nil, err } } + if ais.Spec.AWSSecretName != nil || ais.Spec.GCPSecretName != nil { + if globalConf.Backend.Conf == nil { + globalConf.Backend.Conf = make(map[string]interface{}, 8) + } + if ais.Spec.AWSSecretName != nil { + globalConf.Backend.Conf["aws"] = aisv1.Empty{} + } + if ais.Spec.GCPSecretName != nil { + globalConf.Backend.Conf["gcp"] = aisv1.Empty{} + } + } conf, err := jsoniter.MarshalToString(globalConf) if err != nil { return nil, err diff --git a/operator/pkg/resources/cmn/env.go b/operator/pkg/resources/cmn/env.go index d471a464..89ebf27b 100644 --- a/operator/pkg/resources/cmn/env.go +++ b/operator/pkg/resources/cmn/env.go @@ -27,6 +27,8 @@ const ( EnvEnableExternalAccess = "ENABLE_EXTERNAL_ACCESS" // Bool flag to indicate AIS daemon is exposed using LoadBalancer EnvShutdownMarkerPath = "AIS_SHUTDOWN_MARKER_PATH" // Path where node shutdown marker will be located + EnvGCPCredsPath = "GOOGLE_APPLICATION_CREDENTIALS" // Path to GCP credentials + // Benchmark see: https://github.com/NVIDIA/aistore/blob/master/docs/howto_benchmark.md#dry-run-performance-tests EnvNoDiskIO = "AIS_NO_DISK_IO" EnvDryObjSize = "AIS_DRY_OBJ_SIZE" diff --git a/operator/pkg/resources/cmn/res.go b/operator/pkg/resources/cmn/res.go index 1fca7ead..a4622bbf 100644 --- a/operator/pkg/resources/cmn/res.go +++ b/operator/pkg/resources/cmn/res.go @@ -15,7 +15,7 @@ import ( ) func NewAISVolumes(ais *aisv1.AIStore, daeType string) []corev1.Volume { - return []corev1.Volume{ + volumes := []corev1.Volume{ { Name: "config-mount", VolumeSource: corev1.VolumeSource{ @@ -71,6 +71,28 @@ func NewAISVolumes(ais *aisv1.AIStore, daeType string) []corev1.Volume { }, }, } + if ais.Spec.AWSSecretName != nil { + volumes = append(volumes, corev1.Volume{ + Name: "aws-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: *ais.Spec.AWSSecretName, + }, + }, + }) + } + if ais.Spec.GCPSecretName != nil { + volumes = append(volumes, corev1.Volume{ + Name: "gcp-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: *ais.Spec.GCPSecretName, + }, + }, + }) + } + + return volumes } func NewAISLivenessProbe(port intstr.IntOrString) *corev1.Probe { @@ -97,12 +119,12 @@ func NewAISNodeLifecycle() *corev1.Lifecycle { } } -func NewAISVolumeMounts(antiAffinityDisabled *bool) []corev1.VolumeMount { +func NewAISVolumeMounts(ais *aisv1.AIStore) []corev1.VolumeMount { var hostMountSubPath string - if antiAffinityDisabled != nil && *antiAffinityDisabled { + if ais.Spec.DisablePodAntiAffinity != nil && *ais.Spec.DisablePodAntiAffinity { hostMountSubPath = "$(MY_POD)" } - return []corev1.VolumeMount{ + volumeMounts := []corev1.VolumeMount{ { Name: "config-mount", MountPath: "/var/ais_config", @@ -137,6 +159,23 @@ func NewAISVolumeMounts(antiAffinityDisabled *bool) []corev1.VolumeMount { MountPath: "/var/statsd_config", }, } + + if ais.Spec.AWSSecretName != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "aws-creds", + ReadOnly: true, + MountPath: "/root/.aws", + }) + } + if ais.Spec.GCPSecretName != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "gcp-creds", + ReadOnly: true, + MountPath: "/var/gcp", + }) + } + + return volumeMounts } func NewInitVolumeMounts(antiAffinityDisabled *bool) []corev1.VolumeMount { diff --git a/operator/pkg/resources/proxy/statefulset.go b/operator/pkg/resources/proxy/statefulset.go index 3a9738b8..0954ed38 100644 --- a/operator/pkg/resources/proxy/statefulset.go +++ b/operator/pkg/resources/proxy/statefulset.go @@ -81,6 +81,10 @@ func proxyPodSpec(ais *aisv1.AIStore) corev1.PodSpec { cmn.EnvFromFieldPath(cmn.EnvPublicHostname, "status.hostIP"), } } + if ais.Spec.GCPSecretName != nil { + // TODO -- FIXME: Remove hardcoding for path + optionals = append(optionals, cmn.EnvFromValue(cmn.EnvGCPCredsPath, "/var/gcp/gcp.json")) + } return corev1.PodSpec{ InitContainers: []corev1.Container{ @@ -109,7 +113,7 @@ func proxyPodSpec(ais *aisv1.AIStore) corev1.PodSpec { Name: "ais-node", Image: ais.Spec.NodeImage, ImagePullPolicy: corev1.PullAlways, - Env: []corev1.EnvVar{ + Env: append([]corev1.EnvVar{ cmn.EnvFromFieldPath(cmn.EnvPodName, "metadata.name"), cmn.EnvFromValue(cmn.EnvNS, ais.Namespace), cmn.EnvFromValue(cmn.EnvClusterDomain, ais.GetClusterDomain()), @@ -125,10 +129,10 @@ func proxyPodSpec(ais *aisv1.AIStore) corev1.PodSpec { cmn.EnvFromValue(cmn.EnvProxyServiceName, HeadlessSVCName(ais)), cmn.EnvFromValue(cmn.EnvProxyServicePort, ais.Spec.ProxySpec.ServicePort.String()), cmn.EnvFromValue(cmn.EnvNodeServicePort, ais.Spec.ProxySpec.PublicPort.String()), - }, + }, optionals...), Ports: cmn.NewDaemonPorts(ais.Spec.ProxySpec), SecurityContext: ais.Spec.ProxySpec.ContainerSecurity, - VolumeMounts: cmn.NewAISVolumeMounts(ais.Spec.DisablePodAntiAffinity), + VolumeMounts: cmn.NewAISVolumeMounts(ais), Lifecycle: cmn.NewAISNodeLifecycle(), LivenessProbe: cmn.NewAISLivenessProbe(ais.Spec.ProxySpec.ServicePort), ReadinessProbe: readinessProbe(), diff --git a/operator/pkg/resources/target/statefulset.go b/operator/pkg/resources/target/statefulset.go index a940d0e9..7044ab5a 100644 --- a/operator/pkg/resources/target/statefulset.go +++ b/operator/pkg/resources/target/statefulset.go @@ -47,6 +47,12 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { cmn.EnvFromFieldPath(cmn.EnvPublicHostname, "status.hostIP"), } } + + if ais.Spec.GCPSecretName != nil { + // TODO -- FIXME: Remove hardcoding for path + optionals = append(optionals, cmn.EnvFromValue(cmn.EnvGCPCredsPath, "/var/gcp/gcp.json")) + } + return &apiv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: statefulSetName(ais), @@ -98,7 +104,7 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { Name: "ais-node", Image: ais.Spec.NodeImage, ImagePullPolicy: corev1.PullAlways, - Env: []corev1.EnvVar{ + Env: append([]corev1.EnvVar{ cmn.EnvFromFieldPath(cmn.EnvPodName, "metadata.name"), cmn.EnvFromValue(cmn.EnvClusterDomain, ais.GetClusterDomain()), cmn.EnvFromValue(cmn.EnvNS, ais.Namespace), @@ -118,7 +124,7 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { cmn.EnvFromValue(cmn.EnvProxyServiceName, proxy.HeadlessSVCName(ais)), cmn.EnvFromValue(cmn.EnvProxyServicePort, ais.Spec.ProxySpec.ServicePort.String()), cmn.EnvFromValue(cmn.EnvNodeServicePort, ais.Spec.TargetSpec.PublicPort.String()), - }, + }, optionals...), Ports: cmn.NewDaemonPorts(ais.Spec.TargetSpec.DaemonSpec), SecurityContext: ais.Spec.TargetSpec.ContainerSecurity, VolumeMounts: volumeMounts(ais), @@ -140,7 +146,7 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { } func volumeMounts(ais *aisv1.AIStore) []corev1.VolumeMount { - vols := cmn.NewAISVolumeMounts(ais.Spec.DisablePodAntiAffinity) + vols := cmn.NewAISVolumeMounts(ais) for _, res := range ais.Spec.TargetSpec.Mounts { vols = append(vols, corev1.VolumeMount{ Name: ais.Name + strings.ReplaceAll(res.Path, "/", "-"),