Merge pull request #139 from esune/feat/postgres-custom-schema

Add ability to specify custom schema for verification in postgres
bcgov · Jan 17, 2025 · b198ce5 · b198ce5
2 parents 50e66f5 + 5b17a50
commit b198ce5
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 8 deletions.
diff --git a/config/backup.conf b/config/backup.conf
@@ -41,6 +41,7 @@
 # Full Example:
 # -----------------------------------------------------------
 # postgres=postgresql:5432/TheOrgBook_Database
+# postgres=postgresql:5432/mydb?verifySchema=my_schema
 # mongo=mender-mongodb:27017/useradm
 # postgres=wallet-db/tob_issuer
 # mssql=pims-db-dev:1433/pims

diff --git a/docker/backup.config.utils b/docker/backup.config.utils
@@ -5,15 +5,33 @@
 function getDatabaseName(){
   (
     _databaseSpec=${1}
-    _databaseName=$(echo ${_databaseSpec} | sed -n 's~^.*/\(.*$\)~\1~p')
+    _databaseName=$(echo ${_databaseSpec} | sed -n 's~^.*/\([^?]*\).*~\1~p')
     echo "${_databaseName}"
   )
 }
 
+function getSchema(){
+  (
+    _databaseSpec=${1}
+    _backupMode=${2:-'verify'}
+    _schema=""
+
+    if [ "${_backupMode}" == 'backup' ]; then
+      # Currently not in use, needs more tweaks to be used.
+      # _schema=$(echo ${_databaseSpec} | sed -n  's~^.*backupSchema=\([^&]*\).*~\1~p')
+      echoYellow "Backing up only a specific schema is not supported yet."
+    elif [ "${_backupMode}" == 'verify' ]; then
+      _verifySchema=$(echo ${_databaseSpec} | sed -n  's~^.*verifySchema=\([^&]*\).*~\1~p')
+      _schema="${_verifySchema:-$TABLE_SCHEMA}"      
+    fi
+    echo "${_schema}"
+  )
+}
+
 function getDatabaseType(){
   (
     _databaseSpec=${1}
-    _databaseType=$(echo ${_databaseSpec} | sed -n 's~^\(.*\)=.*$~\1~p' | tr '[:upper:]' '[:lower:]')
+    _databaseType=$(echo ${_databaseSpec} | sed -n 's~^\(.*\)([^=?])~\1~p' | tr '[:upper:]' '[:lower:]')
     echo "${_databaseType}"
   )
 }
@@ -56,7 +74,7 @@ function getHostname(){
     _databaseSpec=${1}
 
     if [ -z "${localhost}" ]; then
-      _hostname=$(echo ${_databaseSpec} | sed 's~^.\+[=]~~;s~[:/].*~~')
+      _hostname=$(echo ${_databaseSpec} | sed 's~^.\+[=\+=[^?]]~~;s~[:/].*~~')
     else
       _hostname="127.0.0.1"
     fi
@@ -114,17 +132,18 @@ function readConf(){
       #  - Remove any lines that do not match the expected database spec format(s)
       #     - [<DatabaseType>=]<Hostname/>/<DatabaseName/>
       #     - [<DatabaseType>=]<Hostname/>:<Port/>/<DatabaseName/>
-      filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/!d;"
+      #     - [<DatabaseType>=]<Hostname/>:<Port/>/<DatabaseName/>?<ExtraParams/>
+      filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*\(\?.*\)\?$/!d;"
       if [ -z "${all}" ]; then
         # Remove any database configs that are not for the current container type
         # Database configs that do not define the database type are assumed to be for the current container type
-        filters+="/\(^[a-zA-Z0-9_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$\)\|\(^${CONTAINER_TYPE}=\)/!d;"
+        filters+="/\(^[a-zA-Z0-9_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*\([^?]*\).*\)\|\(^${CONTAINER_TYPE}=\)/!d;"
       fi
     else
       # Read in the cron config ...
       #  - Remove any lines that MATCH expected database spec format(s),
       #    leaving, what should be, cron tabs.
-      filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/d;"
+      filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*\(\?.*\)\?$/d;"
     fi
 
     if [ -f ${BACKUP_CONF} ]; then

diff --git a/docker/backup.postgres.plugin b/docker/backup.postgres.plugin
@@ -25,12 +25,18 @@ function onBackupDatabase(){
     _portArg=${_port:+"-p ${_port}"}
     _username=$(getUsername ${_databaseSpec})
     _password=$(getPassword ${_databaseSpec})
+
     echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..."
 
     export PGPASSWORD=${_password}
     pg_dump -Fp -h "${_hostname}" ${_portArg} -U "${_username}" "${_database}" > "${BACKUP_DIR}backup.sql"
     pg_dumpall -h "${_hostname}" ${_portArg} -U "${_username}" --roles-only --no-role-passwords > "${BACKUP_DIR}roles.sql"
-    cat "${BACKUP_DIR}roles.sql" "${BACKUP_DIR}backup.sql" | gzip > ${_backupFile}
+
+    # post-process roles.sql to remove the "CREATE ROLE" and "ALTER ROLE" statements for the default roles
+    sed -i "/^CREATE ROLE \"${_username}\";/d; /^CREATE ROLE postgres;/d; /^ALTER ROLE \"${_username}\" /d; /^ALTER ROLE postgres /d" "${BACKUP_DIR}roles.sql"
+
+    # create a single gzip file with both the database and roles
+    cat "${BACKUP_DIR}backup.sql" "${BACKUP_DIR}roles.sql" | gzip > ${_backupFile}
     rm "${BACKUP_DIR}roles.sql" && rm "${BACKUP_DIR}backup.sql"
     return ${PIPESTATUS[0]}
   )
@@ -214,13 +220,14 @@ function onVerifyBackup(){
 
     _hostname=$(getHostname -l ${_databaseSpec})
     _database=$(getDatabaseName ${_databaseSpec})
+    _schema=$(getSchema ${_databaseSpec} 'verify')
     _port=$(getPort -l ${_databaseSpec})
     _portArg=${_port:+"-p ${_port}"}
     _username=$(getUsername ${_databaseSpec})
     _password=$(getPassword ${_databaseSpec})
 
     debugMsg "backup.postgres.plugin - onVerifyBackup"
-    tables=$(psql -h "${_hostname}" ${_portArg} -d "${_database}" -t -c "SELECT table_name FROM information_schema.tables WHERE table_schema='${TABLE_SCHEMA}' AND table_type='BASE TABLE';")
+    tables=$(psql -h "${_hostname}" ${_portArg} -d "${_database}" -t -c "SELECT table_name FROM information_schema.tables WHERE table_schema='${_schema}' AND table_type='BASE TABLE';")
     rtnCd=${?}
 
     # Get the size of the restored database