New enable-internal-pgbr-cmds argument

CHANGELOG.md

@@ -13,6 +13,11 @@ Changelog
   Instead, a WARNING message "max WAL is not the latest archive" will be 
   displayed (suggested by blogh).
   - Fix S3 archives detection (reported by khadijahvf).
+  - New enable-internal-pgbr-cmds argument, for pgBackRest >= 2.28. Internal
+  pgBackRest commands will then be used to list and get the content of files
+  in the repository instead of Perl specific drivers. This is, for instance,
+  needed to access encrypted repositories. This should become the default and
+  only access method in the next release, removing some Perl dependencies.
 
 2020-03-16 v1.8:
 

INSTALL.md

@@ -47,4 +47,10 @@ only be found in the Open Fusion repositories. To install it on CentOS 7, use:
 ```
 yum install -y http://repo.openfusion.net/centos7-x86_64/\
 openfusion-release-0.7-1.of.el7.noarch.rpm
-```
+```
+
+## Unneeded Perl dependencies with pgBackRest >= 2.28
+
+Using the `--enable-internal-pgbr-cmds` argument will use pgBackRest internal
+commands. SFTP or Amazon S3 specific Perl dependencies are then not needed
+anymore.

README

@@ -104,9 +104,9 @@ DESCRIPTION
         unit per given value. If not set, the last unit is in seconds. Eg.
         "1h 55m 6" = "1h55m6s".
 
-        All the missing archives are only shown in the `--debug` mode.
+        All the missing archives are only shown in the "--debug" mode.
 
-        Use `--list-archives` in addition with `--debug` to print the list
+        Use "--list-archives" in addition with "--debug" to print the list
         of all the archived WAL segments.
 
         By default, all the archives older than the oldest backup start
@@ -116,6 +116,10 @@ DESCRIPTION
         Use the "--extended-check" argument to force a full check of the
         found archives and raise warnings in case of inconsistencies.
 
+        After pgBackRest 2.28, it is possible to use the "repo-ls" and
+        "repo-get" internal commands with the "--enable-internal-pgbr-cmds"
+        argument to list and get the content of files in the repository.
+
     check_pgb_version
         Check if this script is running a given version.
 

README.pod

@@ -124,9 +124,9 @@ s (second), m (minute), h (hour), d (day). You can use more than
 one unit per given value. If not set, the last unit is in seconds. 
 Eg. "1h 55m 6" = "1h55m6s".
 
-All the missing archives are only shown in the `--debug` mode.
+All the missing archives are only shown in the C<--debug> mode.
 
-Use `--list-archives` in addition with `--debug` to print the list of all the 
+Use C<--list-archives> in addition with C<--debug> to print the list of all the
 archived WAL segments.
 
 By default, all the archives older than the oldest backup start archive 
@@ -136,6 +136,10 @@ are ignored.
 Use the C<--extended-check> argument to force a full check of the found 
 archives and raise warnings in case of inconsistencies.
 
+After pgBackRest 2.28, it is possible to use the C<repo-ls> and C<repo-get>
+internal commands with the C<--enable-internal-pgbr-cmds> argument to list and
+get the content of files in the repository.
+
 =item B<check_pgb_version>
 
 Check if this script is running a given version.

check_pgbackrest

@@ -449,6 +449,7 @@ sub pgbackrest_get {
     my %args = %{ $args_ref };
     my $filename = shift;
     my $getcmd = $args{'command'}." repo-get";
+    $getcmd .= " --stanza=".$args{'stanza'};
     $getcmd .= " ".$args{'archives_dir'}."/".$filename;
     $getcmd .= " --log-level-console=error";
 
@@ -472,6 +473,7 @@ sub pgbackrest_ls {
     my $args_ref = shift;
     my %args = %{ $args_ref };
     my $lscmd = $args{'command'}." repo-ls";
+    $lscmd .= " --stanza=".$args{'stanza'};
     $lscmd .= " ".$args{'archives_dir'};
     $lscmd .= " --recurse --output=json --log-level-console=error";
 
@@ -645,9 +647,9 @@ s (second), m (minute), h (hour), d (day). You can use more than
 one unit per given value. If not set, the last unit is in seconds. 
 Eg. "1h 55m 6" = "1h55m6s".
 
-All the missing archives are only shown in the `--debug` mode.
+All the missing archives are only shown in the C<--debug> mode.
 
-Use C<--list-archives> in addition with `--debug` to print the list of all the 
+Use C<--list-archives> in addition with C<--debug> to print the list of all the
 archived WAL segments.
 
 By default, all the archives older than the oldest backup start archive 
@@ -657,6 +659,10 @@ are ignored.
 Use the C<--extended-check> argument to force a full check of the found 
 archives and raise warnings in case of inconsistencies.
 
+After pgBackRest 2.28, it is possible to use the C<repo-ls> and C<repo-get>
+internal commands with the C<--enable-internal-pgbr-cmds> argument to list and
+get the content of files in the repository.
+
 =cut
 
 sub get_archived_wal_list {
@@ -677,9 +683,7 @@ sub get_archived_wal_list {
     my $history_re_full = qr/$end_tl.history$/;
 
     my $pgbackrest_version=pgbackrest_version(\%args);
-    my $activate_pgbackrest_ls_command=0;
-    if($pgbackrest_version >= '2.28' && $activate_pgbackrest_ls_command){
-        # pgBackRest ls command
+    if($pgbackrest_version >= '2.28' && $args{'enable-internal-pgbr-cmds'}){
         my $list = pgbackrest_ls(\%args);
 
         foreach my $key (keys %{$list}) {
@@ -1225,6 +1229,7 @@ GetOptions(
         'command|C=s',
         'config|c=s',
         'debug!',
+        'enable-internal-pgbr-cmds!',
         'extended-check!',
         'help|?!',
         'ignore-archived-after=s',
@@ -1289,6 +1294,12 @@ pod2usage(
 ) if ( $args{'extended-check'} or $args{'ignore-archived-after'} or $args{'ignore-archived-before'} or $args{'latest-archive-age-alert'} )
     and $args{'service'} ne 'archives';
 
+# Check "archives" specific arg --enable-internal-pgbr-cmds
+pod2usage(
+    -message => 'FATAL: "enable-internal-pgbr-cmds" is only allowed with "archives" service.',
+    -exitval => 127
+) if $args{'enable-internal-pgbr-cmds'} and $args{'service'} ne 'archives';
+
 # Check "archives" specific arg --list-archives
 pod2usage(
     -message => 'FATAL: "list-archives" is only allowed with "archives" service and "debug" option.',

test/VALIDATION.md

@@ -0,0 +1,22 @@
+# Validation process
+
+```bash
+time sudo make s1_light
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/simulate-extended-activity.bash -s 10 -a 10 -p local"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s1.bash -s -p /check_pgbackrest"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s1.bash -s -p /check_pgbackrest -a '--enable-internal-pgbr-cmds'"
+```
+
+```bash
+time sudo make s2_light
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/simulate-extended-activity.bash -s 10 -a 10 -p remote"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s2-from-primary.bash -s -p /check_pgbackrest"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s2-from-primary.bash -s -p /check_pgbackrest -a '--enable-internal-pgbr-cmds'"
+```
+
+```bash
+time sudo make s3_light
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/simulate-extended-activity.bash -s 10 -a 10 -p local"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s3.bash -s -p /check_pgbackrest"
+time sudo vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s3.bash -s -p /check_pgbackrest -a '--enable-internal-pgbr-cmds'"
+```

test/Vagrantfile

@@ -1,5 +1,6 @@
 pgver = '12' # pg version to use
 vm_prefix = 'c_pgbr_c7'
+encrypted_repo = 'false'
 
 Vagrant.configure(2) do |config|
 
@@ -66,7 +67,7 @@ Vagrant.configure(2) do |config|
         # pgbackrest local setup. Use "vagrant up --provision-with=pgbackrest_local"
         pgsql.vm.provision 'pgbackrest_local', type: 'shell',
           path: 'provision/pgbackrest_local.bash',
-            args: [ pgver, pgdata ],
+            args: [ pgver, pgdata, encrypted_repo ],
             run: 'never'
 
         # pgbackrest remote setup. Use "vagrant up --provision-with=pgbackrest_remote_primary"
@@ -78,7 +79,7 @@ Vagrant.configure(2) do |config|
         # pgbackrest minio setup. Use "vagrant up --provision-with=pgbackrest_minio"
         pgsql.vm.provision 'pgbackrest_minio', type: 'shell',
           path: 'provision/pgbackrest_minio.bash',
-            args: [ pgver, pgdata ],
+            args: [ pgver, pgdata, encrypted_repo ],
             run: 'never'
     end
 
@@ -92,7 +93,7 @@ Vagrant.configure(2) do |config|
         # pgbackrest remote setup. Use "vagrant up --provision-with=pgbackrest_remote_standby"
         backup.vm.provision 'pgbackrest_remote_standby', type: 'shell',
           path: 'provision/pgbackrest_remote_standby.bash',
-            args: [ pgver, pgdata ],
+            args: [ pgver, pgdata, encrypted_repo ],
             run: 'never'
 
         # minio local setup. Use "vagrant up --provision-with=minio"

test/provision/pgbackrest_local.bash

@@ -6,6 +6,7 @@ set -o pipefail
 
 PGVER="$1"
 PGDATA="$2"
+ENCRYPTED="$3"
 
 PACKAGES=(
     pgbackrest
@@ -28,7 +29,13 @@ EOF
 chmod 755 /var/lib/pgbackrest
 mount /var/lib/pgbackrest
 
+CIPHER=
 # pgbackrest.conf setup
+if [ $ENCRYPTED = "true" ]; then
+    CIPHER='repo1-cipher-type=aes-256-cbc
+repo1-cipher-pass=acbd'
+fi
+
 cat<<EOC > "/etc/pgbackrest.conf"
 [global]
 repo1-type=cifs
@@ -39,8 +46,7 @@ log-level-console=warn
 log-level-file=info
 start-fast=y
 delta=y
-repo1-cipher-type=aes-256-cbc
-repo1-cipher-pass=acbd
+$CIPHER
 
 [my_stanza]
 pg1-path=${PGDATA}
@@ -54,8 +60,4 @@ ALTER SYSTEM SET "archive_command" TO 'pgbackrest --stanza=my_stanza archive-pus
 SELECT pg_reload_conf();
 EOS
 
-sudo -iu postgres pgbackrest --stanza=my_stanza check
-
-
-
-
+sudo -iu postgres pgbackrest --stanza=my_stanza check

test/provision/pgbackrest_minio.bash

@@ -6,14 +6,21 @@ set -o pipefail
 
 PGVER="$1"
 PGDATA="$2"
+ENCRYPTED="$3"
 
 PACKAGES=(
     pgbackrest
 )
 
 yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}"
 
+CIPHER=
 # pgbackrest.conf setup
+if [ $ENCRYPTED = "true" ]; then
+    CIPHER='repo1-cipher-type=aes-256-cbc
+repo1-cipher-pass=acbd'
+fi
+
 cat<<EOC > "/etc/pgbackrest.conf"
 [global]
 repo1-path=/repo1
@@ -31,8 +38,7 @@ log-level-console=warn
 log-level-file=info
 start-fast=y
 delta=y
-repo1-cipher-type=aes-256-cbc
-repo1-cipher-pass=acbd
+$CIPHER
 
 [my_stanza]
 pg1-path=${PGDATA}

test/provision/pgbackrest_remote_primary.bash

@@ -31,4 +31,4 @@ EOC
 cat <<'EOS' | "/usr/pgsql-${PGVER}/bin/psql" -U postgres
 ALTER SYSTEM SET "archive_command" TO 'pgbackrest --stanza=my_stanza archive-push %p';
 SELECT pg_reload_conf();
-EOS
+EOS

test/provision/pgbackrest_remote_standby.bash

@@ -6,14 +6,21 @@ set -o pipefail
 
 PGVER="$1"
 PGDATA="$2"
+ENCRYPTED="$3"
 
 PACKAGES=(
     pgbackrest
 )
 
 yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}"
 
+CIPHER=
 # pgbackrest.conf setup
+if [ $ENCRYPTED = "true" ]; then
+    CIPHER='repo1-cipher-type=aes-256-cbc
+repo1-cipher-pass=acbd'
+fi
+
 cat<<EOC > "/etc/pgbackrest.conf"
 [global]
 repo1-path=/var/lib/pgbackrest
@@ -23,8 +30,7 @@ log-level-console=warn
 log-level-file=info
 start-fast=y
 delta=y
-repo1-cipher-type=aes-256-cbc
-repo1-cipher-pass=acbd
+$CIPHER
 
 [my_stanza]
 pg1-host=pgsql-srv
@@ -46,8 +52,7 @@ process-max=2
 log-level-console=warn
 log-level-file=info
 delta=y
-repo1-cipher-type=aes-256-cbc
-repo1-cipher-pass=acbd
+$CIPHER
 
 [my_stanza]
 pg1-path=${PGDATA}

test/provision/pgsql.bash

@@ -10,6 +10,10 @@ PGDATA="$2"
 # install required packages
 yum install --nogpgcheck --quiet -y -e 0 "https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm"
 
+if [ $PGVER -ge "13" ]; then
+    yum-config-manager --enable pgdg$PGVER-updates-testing
+fi
+
 PACKAGES=(
     "postgresql${PGVER}"
     "postgresql${PGVER}-server"
@@ -70,4 +74,4 @@ usermod -aG postgres accessed_by_ssh
 echo "accessed_by_ssh ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 cp -rf /root/.ssh /home/accessed_by_ssh/.ssh
 chown -R accessed_by_ssh: /home/accessed_by_ssh/.ssh
-restorecon -R /home/accessed_by_ssh/.ssh
+restorecon -R /home/accessed_by_ssh/.ssh