copy gen3 dataabses in Aurora (#2356)

* copy gen3 dataabses in Aurora * Update psql-db-copy-aurora-job.yaml * Update psql-db-copy-aurora-job.yaml * add service account * Add documentation for dbbackup.sh * Add psql-db-aurora-migration-job, updating dbbackup.sh * Add psql-db-aurora-migration-job, updating dbbackup.sh * Update dbbackup.sh * Update psql-db-copy-aurora-job.yaml * Update dbbackup.md
uc-cdis · Jul 17, 2024 · e7fb972 · e7fb972
1 parent 2f7e8aa
commit e7fb972
Show file tree

Hide file tree

Showing 5 changed files with 575 additions and 21 deletions.
diff --git a/doc/dbbackup.md b/doc/dbbackup.md
@@ -0,0 +1,52 @@
+# TL;DR
+
+This script facilitates the management of database backup and restore within the Gen3 environment. It can establish policies, service accounts, roles, and S3 buckets. Depending on the command provided, it can initiate a database dump, perform a restore, migrate databases to a new RDS instance on Aurora, or clone databases to an RDS Aurora instance.
+
+## Usage
+
+```sh
+gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora]
+```
+
+### Commands
+
+#### dump
+
+Initiates a database dump and pushes it to an S3 bucket, creating the essential AWS resources if they are absent. The dump operation is intended to be executed from the namespace/commons that requires the backup.
+
+```sh
+gen3 dbbackup dump
+```
+
+#### restore
+
+Initiates a database restore from an S3 bucket, creating the essential AWS resources if they are absent. The restore operation is meant to be executed in the target namespace where the backup needs to be restored.
+
+```sh
+gen3 dbbackup restore
+```
+
+#### create-sa
+
+Creates the necessary service account and roles for DB copy.
+
+```sh
+gen3 dbbackup create-sa
+```
+
+#### migrate-to-aurora
+
+Triggers a service account creation and a job to migrate a Gen3 commons to an AWS RDS Aurora instance.
+
+```sh
+gen3 dbbackup migrate-to-aurora
+```
+
+#### copy-to-aurora
+
+Triggers a service account creation and a job to copy the databases Indexd, Sheepdog & Metadata to new databases within an RDS Aurora cluster from another namespace <source-namespace> in same RDS cluster.
+
+```sh
+gen3 dbbackup copy-to-aurora <source-namespace>
+```
+
diff --git a/gen3/bin/dbbackup.sh b/gen3/bin/dbbackup.sh
@@ -1,26 +1,28 @@
 #!/bin/bash
 
 ####################################################################################################
-# Script: dbdump.sh
+# Script: dbbackup.sh
 #
 # Description:
 #   This script facilitates the management of database backups within the gen3 environment. It is
-#   equipped to establish policies, service accounts, roles, and S3 buckets. Depending on the 
-#   command provided, it will either initiate a database dump or perform a restore.
+#   equipped to establish policies, service accounts, roles, and S3 buckets. Depending on the
+#   command provided, it will either initiate a database dump, perform a restore, migrate to Aurora,
+#   or copy to Aurora.
 #
 # Usage:
-#   gen3 dbbackup [dump|restore]
+#   gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora <source_namespace>]
 #
-#   dump    - Initiates a database dump, creating the essential AWS resources if they are absent.
-#             The dump operation is intended to be executed from the namespace/commons that requires
-#             the backup.
-#   restore - Initiates a database restore, creating the essential AWS resources if they are absent.
-#             The restore operation is meant to be executed in the target namespace, where the backup 
-#             needs to be restored.
+#   dump           - Initiates a database dump, creating the essential AWS resources if they are absent.
+#                    The dump operation is intended to be executed from the namespace/commons that requires
+#                    the backup.
+#   restore        - Initiates a database restore, creating the essential AWS resources if they are absent.
+#                    The restore operation is meant to be executed in the target namespace, where the backup
+#                    needs to be restored.
+#   va-dump        - Runs a va-testing DB dump.
+#   create-sa      - Creates the necessary service account and roles for DB copy.
+#   migrate-to-aurora - Triggers a service account creation and a job to migrate a Gen3 commons to an AWS RDS Aurora instance.
+#   copy-to-aurora    - Triggers a service account creation and a job to copy the databases Indexd, Sheepdog & Metadata to new databases within an RDS Aurora cluster.
 #
-# Notes:
-#   This script extensively utilizes the AWS CLI and the gen3 CLI. Proper functioning demands a 
-#   configured gen3 environment and the availability of the necessary CLI tools.
 #
 ####################################################################################################
 
@@ -49,7 +51,6 @@ gen3_log_info "namespace: $namespace"
 gen3_log_info "sa_name: $sa_name"
 gen3_log_info "bucket_name: $bucket_name"
 
-
 # Create an S3 access policy if it doesn't exist
 create_policy() {
   # Check if policy exists
@@ -87,7 +88,6 @@ EOM
   fi
 }
 
-
 # Create or update the Service Account and its corresponding IAM Role
 create_service_account_and_role() {
     cluster_arn=$(kubectl config current-context)
@@ -101,7 +101,6 @@ create_service_account_and_role() {
     gen3_log_info "oidc_url: $oidc_url"
     gen3_log_info "role_name: $role_name"
 
-
   cat > ${trust_policy} <<EOF
 {
   "Version": "2012-10-17",
@@ -161,13 +160,11 @@ create_s3_bucket() {
   fi
 }
 
-
 # Function to trigger the database backup job
 db_dump() {
     gen3 job run psql-db-prep-dump
 }
 
-
 # Function to trigger the database backup restore job
 db_restore() {
     gen3 job run psql-db-prep-restore
@@ -177,8 +174,55 @@ va_testing_db_dump() {
   gen3 job run psql-db-dump-va-testing
 }
 
+# Function to create the psql-db-copy service account and roles
+create_db_copy_service_account() {
+  cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: psql-db-copy-sa
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psql-db-copy-role
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "watch", "list"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: psql-db-copy-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psql-db-copy-role
+subjects:
+- kind: ServiceAccount
+  name: psql-db-copy-sa
+  namespace: ${namespace}
+EOF
+}
+
+# Function to run the Aurora migration job
+migrate_to_aurora() {
+    create_db_copy_service_account
+    sleep 30
+    gen3 job run psql-db-aurora-migration
+}
+
+# Function to run the Aurora copy job
+copy_to_aurora() {
+    create_db_copy_service_account
+    sleep 30
+    gen3 job run psql-db-copy-aurora SOURCE_NAMESPACE "$1"
+}
 
-# main function to determine whether dump or restore
+# main function to determine whether dump, restore, or create service account
 main() {
     case "$1" in
         dump)
@@ -202,11 +246,27 @@ main() {
             create_s3_bucket
             va_testing_db_dump
             ;;
+        create-sa)
+            gen3_log_info "Creating service account for DB copy..."
+            create_db_copy_service_account
+            ;;
+        migrate-to-aurora)
+            gen3_log_info "Migrating Gen3 commons to Aurora..."
+            migrate_to_aurora
+            ;;
+        copy-to-aurora)
+            if [ -z "$2" ]; then
+                echo "Usage: $0 copy-to-aurora <source_namespace>"
+                exit 1
+            fi
+            gen3_log_info "Copying databases within Aurora..."
+            copy_to_aurora "$2"
+            ;;
         *)
-            echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump]"
+            echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora <source_namespace>]"
             return 1
             ;;
     esac
 }
 
-main "$1"
+main "$@"