From 71df947318e806c132fc38eb25940fffb670b9f9 Mon Sep 17 00:00:00 2001
From: Barni S <barni@google.com>
Date: Mon, 27 Jan 2025 20:12:40 -0500
Subject: [PATCH] Add CloudSQL example with single primary and replica

---
 examples/gcp/README.md                        | 130 ++++++++++++
 examples/gcp/cloud-sql/README.md              |  64 ++++++
 examples/gcp/cloud-sql/instance-template.yaml |  10 +
 examples/gcp/cloud-sql/rg.yaml                | 198 ++++++++++++++++++
 4 files changed, 402 insertions(+)
 create mode 100644 examples/gcp/README.md
 create mode 100644 examples/gcp/cloud-sql/README.md
 create mode 100644 examples/gcp/cloud-sql/instance-template.yaml
 create mode 100644 examples/gcp/cloud-sql/rg.yaml

diff --git a/examples/gcp/README.md b/examples/gcp/README.md
new file mode 100644
index 00000000..b3527513
--- /dev/null
+++ b/examples/gcp/README.md
@@ -0,0 +1,130 @@
+# Getting Started with GCP
+
+## Prerequisites
+
+1. Kubectl
+
+   1. Install kubectl on [macos](https://kubernetes.io/docs/tasks/tools/install-kubectl-macos/) or [linux](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) 
+
+2. Gcloud
+
+   1. To install gcloud, please see [here](https://cloud.google.com/sdk/docs/install)
+
+   2. [initialize](https://cloud.google.com/sdk/docs/initializing) the gcloud CLI
+
+## Create a new Google Cloud project
+
+Start by creating a new Google Cloud (GCP) project. When trying out KRO, we suggested that you use a separate project to avoid disrupting any production clusters or services. You may choose to follow your own best practices in setting up the project.
+
+Steps to setup a new GCP project: 
+
+```
+# in *nix shell USER should be set. if not set USERNAME explicitly
+export USERNAME=${USER?}
+export PROJECT_ID=kro-${USERNAME?} 
+export REGION=us-central1 # << CHANGE region here 
+# Please set the appropriate folder, billing
+export GCP_FOLDER=0000000000
+export GCP_BILLING=000000-000000-000000
+export ADMINUSER=someone@company.com
+
+# Separate Gcloud configuration
+gcloud config configurations create kro
+gcloud config configurations activate kro
+gcloud config set account ${ADMINUSER?}
+
+# Create the project
+gcloud projects create ${PROJECT_ID?} --folder=${GCP_FOLDER?}
+gcloud auth application-default set-quota-project ${PROJECT_ID?}
+
+# attach billing (THIS IS IMPORTANT)
+gcloud beta billing projects link ${PROJECT_ID?} --billing-account ${GCP_BILLING?}
+
+# Set the project ID in the current configuration
+gcloud config set project ${PROJECT_ID?}
+```
+
+## Enable Google Cloud services
+
+Enable the following required APIs:
+
+```
+gcloud services enable \
+  container.googleapis.com  \
+  cloudresourcemanager.googleapis.com \
+  serviceusage.googleapis.com
+```
+
+## GKE Cluster with KCC and KRO
+
+1. Create GKE Cluster:
+   1. Navigate to the GCP console [Create Standard GKE cluster](https://pantheon.corp.google.com/kubernetes/add) page.
+   1.  Set the name of the cluster (to say "kro")
+   1. Enable Workload Identity under Cluster-Security settings. 
+   1. Click create cluster. This step takes 5-10 mins.
+2. Install KCC using these [instructions](https://cloud.google.com/config-connector/docs/how-to/install-manually):
+   1. [Install KCC](https://cloud.google.com/config-connector/docs/how-to/install-manually#installing_the_operator)  
+   ```
+   gcloud storage cp gs://configconnector-operator/latest/release-bundle.tar.gz release-bundle.tar.gz
+   tar zxvf release-bundle.tar.gz
+   kubectl apply -f operator-system/configconnector-operator.yaml
+   ```
+   2. [Create SA and bind with KSA](https://cloud.google.com/config-connector/docs/how-to/install-manually#identity) with KCC  
+   ```
+   gcloud iam service-accounts create kcc-operator
+   gcloud projects add-iam-policy-binding ${PROJECT_ID}\
+    --member="serviceAccount:kcc-operator@${PROJECT_ID}.iam.gserviceaccount.com" \
+    --role="roles/editor"
+   gcloud iam service-accounts add-iam-policy-binding kcc-operator@${PROJECT_ID}.iam.gserviceaccount.com \
+    --member="serviceAccount:${PROJECT_ID}.svc.id.goog[cnrm-system/cnrm-controller-manager]" \
+    --role="roles/iam.workloadIdentityUser"
+   ```
+   3. [Configure Config Connector](https://cloud.google.com/config-connector/docs/how-to/install-manually#addon-configuring)  
+   ```
+   kubectl apply -f - <<EOF
+   apiVersion: core.cnrm.cloud.google.com/v1beta1
+   kind: ConfigConnector
+   metadata:
+     name: configconnector.core.cnrm.cloud.google.com
+   spec:
+     mode: cluster
+     googleServiceAccount: "kcc-operator@${PROJECT_ID?}.iam.gserviceaccount.com"
+     stateIntoSpec: Absent
+   EOF
+   ```
+   4. [Create a namespace](https://cloud.google.com/config-connector/docs/how-to/install-manually#specify) for KCC objects  
+   ```
+   kubectl create namespace config-connector
+   kubectl annotate namespace config-connector cnrm.cloud.google.com/project-id=${PROJECT_ID?}
+   ```
+   5. Verify KCC Installation
+   ```
+   kubectl wait -n cnrm-system --for=condition=Ready pod --all
+   ```
+3. Setup Kubectl to target the cluster `gcloud container clusters get-credentials kro --region $REGION --project $PROJECT_ID`
+4. Install KRO following [instructions here](https://kro.run/docs/getting-started/Installation/)
+
+## Cleanup
+
+When you are done with all the chapters and want to cleanup the environment, follow these steps:
+
+```
+# in *nix shell USER should be set. if not set USERNAME explicitly
+export USERNAME=${USER?}
+export PROJECT_ID=compositions-${USERNAME?}
+export REGION=us-west2
+export CONFIG_CONTROLLER_NAME=compositions
+
+# Delete the GCP project if you created one for trying out KRO
+gcloud projects delete ${PROJECT_ID?}
+# Delete gcloud configuration
+gcloud config configurations activate <anything other than 'compositions'>
+gcloud config configurations delete compositions
+# Delete kubectl context
+kubectl config  delete-context \
+gke_${PROJECT_ID?}_${REGION?}_krmapihost-${CONFIG_CONTROLLER_NAME?}
+
+# Dont forget to switch to another context 
+kubectl config  get-contexts
+kubectl config  use-context <context name>
+```
\ No newline at end of file
diff --git a/examples/gcp/cloud-sql/README.md b/examples/gcp/cloud-sql/README.md
new file mode 100644
index 00000000..22b8b776
--- /dev/null
+++ b/examples/gcp/cloud-sql/README.md
@@ -0,0 +1,64 @@
+# CloudSQL example
+
+This example creates a ResourceGraphDefinition called `CloudSQL` to deploy Cloud SQL instance in 2 regions as a primary replica pair.
+
+## Create ResourceGraphDefinitions
+
+Apply the RGD to your cluster:
+
+```
+kubectl apply -f rg.yaml
+```
+
+Validate the RG:
+
+```
+> kubectl get rg cloudsql.kro.run
+NAME               APIVERSION   KIND       STATE    AGE
+cloudsql.kro.run   v1alpha1     CloudSQL   Active   44m
+```
+
+## Create an Instance of CloudSQL
+Set the env variables used in the instance template:
+```
+export CLOUDSQL_NAME=demo
+export GCP_PROJECT=myproject
+export PRIMARY_REGION=us-central1
+export REPLICA_REGION=us-west1
+```
+
+Run the following command to replace the env variables in `instance-template.yaml` file and create
+a new file called instance.yaml. 
+```shell
+envsubst < "instance-template.yaml" > "instance.yaml"
+```
+
+Apply the `instance.yaml` 
+
+```
+kubectl apply -f instance.yaml
+```
+
+Validate instance status:
+
+```
+kubectl get cloudsqls
+```
+
+## Validate
+
+Navigate to CloudSQL page in the GCP Console and verify the creation of primary and replica instances.
+
+## Clean up
+
+Remove the instance:
+
+```
+kubectl delete cloudsql $CLOUDSQL_NAME
+```
+
+Remove the ResourceGraphDefinitions:
+
+```
+kubectl delete rgd cloudsql.kro.run
+```
diff --git a/examples/gcp/cloud-sql/instance-template.yaml b/examples/gcp/cloud-sql/instance-template.yaml
new file mode 100644
index 00000000..4af6ee8e
--- /dev/null
+++ b/examples/gcp/cloud-sql/instance-template.yaml
@@ -0,0 +1,10 @@
+## CloudSQL
+apiVersion: kro.run/v1alpha1
+kind: CloudSQL
+metadata:
+  name: $CLOUDSQL_NAME
+spec:
+  name: $CLOUDSQL_NAME
+  project: $GCP_PROJECT
+  primaryRegion: $PRIMARY_REGION
+  replicaRegion: $REPLICA_REGION
\ No newline at end of file
diff --git a/examples/gcp/cloud-sql/rg.yaml b/examples/gcp/cloud-sql/rg.yaml
new file mode 100644
index 00000000..17f368e5
--- /dev/null
+++ b/examples/gcp/cloud-sql/rg.yaml
@@ -0,0 +1,198 @@
+apiVersion: kro.run/v1alpha1
+kind: ResourceGroup
+metadata:
+  name: cloudsql.kro.run
+spec:
+  schema:
+    apiVersion: v1alpha1
+    kind: CloudSQL
+    spec:
+      name: string
+      project: string
+      primaryRegion: string
+      replicaRegion: string
+    status:
+      connectionName: ${sqlPrimary.status.connectionName}
+      dnsName: ${sqlPrimary.status.dnsName}
+
+  resources:
+  - id: cloudkmsEnable
+    template:
+      apiVersion: serviceusage.cnrm.cloud.google.com/v1beta1
+      kind: Service
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: "abandon"
+          cnrm.cloud.google.com/disable-dependent-services: "false"
+        name: cloudkms-enablement
+      spec:
+        resourceID: cloudkms.googleapis.com
+  - id: iamEnable
+    template:
+      apiVersion: serviceusage.cnrm.cloud.google.com/v1beta1
+      kind: Service
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: "abandon"
+          cnrm.cloud.google.com/disable-dependent-services: "false"
+        name: iam-enablement
+      spec:
+        resourceID: iam.googleapis.com
+  - id: servceusageEnable
+    template:
+      apiVersion: serviceusage.cnrm.cloud.google.com/v1beta1
+      kind: Service
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: "abandon"
+          cnrm.cloud.google.com/disable-dependent-services: "false"
+        name: serviceusage-enablement
+      spec:
+        resourceID: serviceusage.googleapis.com
+  - id: sqladminEnable
+    template:
+      apiVersion: serviceusage.cnrm.cloud.google.com/v1beta1
+      kind: Service
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: "abandon"
+          cnrm.cloud.google.com/disable-dependent-services: "false"
+        name: sqladmin-enablement
+      spec:
+        resourceID: sqladmin.googleapis.com
+  - id: serviceidentity
+    template:
+      apiVersion: serviceusage.cnrm.cloud.google.com/v1beta1
+      kind: ServiceIdentity
+      metadata:
+        name: sqladmin.googleapis.com
+      spec:
+        projectRef:
+          external: ${schema.spec.project}
+  - id: keyringPrimary
+    template:
+      apiVersion: kms.cnrm.cloud.google.com/v1beta1
+      kind: KMSKeyRing
+      metadata:
+        name: ${schema.spec.name}-primary
+      spec:
+        location: ${schema.spec.primaryRegion}
+  - id: keyringReplica
+    template:
+      apiVersion: kms.cnrm.cloud.google.com/v1beta1
+      kind: KMSKeyRing
+      metadata:
+        name: ${schema.spec.name}-replica
+      spec:
+        location: ${schema.spec.replicaRegion}
+  - id: kmskeyPrimary
+    template:
+      apiVersion: kms.cnrm.cloud.google.com/v1beta1
+      kind: KMSCryptoKey
+      metadata:
+        labels:
+          failure-zone: ${schema.spec.primaryRegion}
+        name: ${schema.spec.name}-primary
+      spec:
+        keyRingRef:
+          name: ${schema.spec.name}-primary
+          #namespace: {{ cloudsqls.metadata.namespace }}
+        purpose: ENCRYPT_DECRYPT
+        versionTemplate:
+          algorithm: GOOGLE_SYMMETRIC_ENCRYPTION
+          protectionLevel: SOFTWARE
+        importOnly: false
+  - id: kmskeyReplica
+    template:
+      apiVersion: kms.cnrm.cloud.google.com/v1beta1
+      kind: KMSCryptoKey
+      metadata:
+        labels:
+          failure-zone: ${schema.spec.replicaRegion}
+        name: ${schema.spec.name}-replica
+      spec:
+        keyRingRef:
+          name: ${schema.spec.name}-replica
+          #namespace: {{ cloudsqls.metadata.namespace }}
+        purpose: ENCRYPT_DECRYPT
+        versionTemplate:
+          algorithm: GOOGLE_SYMMETRIC_ENCRYPTION
+          protectionLevel: SOFTWARE
+        importOnly: false
+  - id: iampolicymemberPrimary
+    template:
+      apiVersion: iam.cnrm.cloud.google.com/v1beta1
+      kind: IAMPolicyMember
+      metadata:
+        name: sql-kms-${schema.spec.primaryRegion}-policybinding
+      spec:
+        member: serviceAccount:${serviceidentity.status.email}
+        role: roles/cloudkms.cryptoKeyEncrypterDecrypter
+        resourceRef:
+          kind: KMSCryptoKey
+          name: ${schema.spec.name}-primary
+          #namespace: {{ cloudsqls.metadata.namespace }}
+  - id: iampolicymemberReplica
+    template:
+      apiVersion: iam.cnrm.cloud.google.com/v1beta1
+      kind: IAMPolicyMember
+      metadata:
+        name: sql-kms-${schema.spec.replicaRegion}-policybinding
+      spec:
+        member: serviceAccount:${serviceidentity.status.email}
+        role: roles/cloudkms.cryptoKeyEncrypterDecrypter
+        resourceRef:
+          kind: KMSCryptoKey
+          name: ${schema.spec.name}-replica
+          #namespace: {{ cloudsqls.metadata.namespace }}
+  - id: sqlPrimary
+    template:
+      apiVersion: sql.cnrm.cloud.google.com/v1beta1
+      kind: SQLInstance
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: abandon
+        labels:
+          failure-zone: ${schema.spec.primaryRegion}
+        name: ${schema.spec.name}-primary
+      spec:
+        databaseVersion: POSTGRES_13
+        encryptionKMSCryptoKeyRef:
+          external: projects/${schema.spec.project}/locations/${schema.spec.primaryRegion}/keyRings/${schema.spec.name}-primary/cryptoKeys/${schema.spec.name}-primary
+        region: ${schema.spec.primaryRegion}
+        settings:
+          availabilityType: REGIONAL
+          backupConfiguration:
+            backupRetentionSettings:
+              retainedBackups: 6
+            enabled: true
+            location: us
+          diskSize: 50
+          diskType: PD_SSD
+          maintenanceWindow:
+            day: 7
+            hour: 3
+          tier: db-custom-8-30720
+  - id: sqlReplica
+    template:
+      apiVersion: sql.cnrm.cloud.google.com/v1beta1
+      kind: SQLInstance
+      metadata:
+        annotations:
+          cnrm.cloud.google.com/deletion-policy: abandon
+        labels:
+          failure-zone: ${schema.spec.replicaRegion}
+        name: ${schema.spec.name}-replica
+      spec:
+        databaseVersion: POSTGRES_13
+        encryptionKMSCryptoKeyRef:
+          external: projects/${schema.spec.project}/locations/${schema.spec.replicaRegion}/keyRings/${schema.spec.name}-replica/cryptoKeys/${schema.spec.name}-replica
+        masterInstanceRef:
+          name: ${schema.spec.name}-primary
+          #namespace: {{ cloudsqls.metadata.namespace }}
+        region: ${schema.spec.primaryRegion}
+        settings:
+          availabilityType: REGIONAL
+          diskSize: 50
+          diskType: PD_SSD
+          tier: db-custom-8-30720
\ No newline at end of file