Create test scenarios with pgbackrest S3 support, using vagrant and m…

…inio

test/Makefile

@@ -1,5 +1,6 @@
 s1: clean create_vm pgbackrest_local
 s2: clean create_vm pgbackrest_remote
+s3: clean create_vm pgbackrest_minio
 
 create_vm:
 	vagrant up
@@ -13,5 +14,10 @@ pgbackrest_remote:
 	vagrant up --provision-with=pgbackrest_remote_standby
 	vagrant up --provision-with=icinga2_remote
 
+pgbackrest_minio:
+	vagrant up --provision-with=minio
+	vagrant up --provision-with=pgbackrest_minio
+	vagrant up --provision-with=icinga2_minio
+
 clean:
 	vagrant destroy -f

test/README.md

@@ -25,6 +25,12 @@ remotely (on `pgsql-srv`). Checks are performed from `icinga-srv` by ssh.
 pgBackRest backups are use to build a Streaming Replication with `backup-srv` 
 as standby server.
 
+### 3. pgBackRest configured to backup and archive to a MinIO S3 bucket
+
+  * `icinga-srv` executes check_pgbackrest by ssh with Icinga 2;
+  * `pgsql-srv` hosting a pgsql cluster with pgBackRest installed;
+  * `backup-srv` hosting the MinIO server.
+
 ## Testing
 
 The easiest way to start testing is with the included `Makefile`.
@@ -75,6 +81,33 @@ vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s2-from-prima
 
 Expected run time: 30 sec.
 
+### Test case 3
+
+_Build_:
+
+```bash
+cd test
+make s3
+```
+
+Expected make time: 7 min.
+
+_Check the results of a manual execution of check_pgbackrest_:
+
+```bash
+vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s3.bash"
+```
+
+Expected run time: 40 sec.
+
+_To simulate some activity with pgBackRest_:
+
+```bash
+vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/simulate-activity-local.bash"
+```
+
+Modify `SCALE` factor or `ACTIVITY_TIME` in this script to simulate more activity.
+
 ### Icinga 2 connectivity
 
 _Check the results of _check_pgbackrest_ launched by Icinga 2:

test/Vagrantfile

@@ -50,6 +50,10 @@ Vagrant.configure(2) do |config|
         # pgbackrest remote setup. Use "vagrant up --provision-with=icinga2_remote"
         icinga.vm.provision 'icinga2_remote', type: 'shell',
           path: 'provision/icinga2_remote.bash', run: 'never'
+
+        # pgbackrest minio setup. Use "vagrant up --provision-with=icinga2_minio"
+        icinga.vm.provision 'icinga2_minio', type: 'shell',
+          path: 'provision/icinga2_minio.bash', run: 'never'
     end
 
     config.vm.define "pgsql-srv" do |pgsql|
@@ -66,6 +70,12 @@ Vagrant.configure(2) do |config|
           path: 'provision/pgbackrest_remote_primary.bash',
             args: [ pgver, pgdata ],
             run: 'never'
+
+        # pgbackrest minio setup. Use "vagrant up --provision-with=pgbackrest_minio"
+        pgsql.vm.provision 'pgbackrest_minio', type: 'shell',
+          path: 'provision/pgbackrest_minio.bash',
+            args: [ pgver, pgdata ],
+            run: 'never'
     end
 
     config.vm.define "backup-srv" do |backup|
@@ -76,6 +86,10 @@ Vagrant.configure(2) do |config|
           path: 'provision/pgbackrest_remote_standby.bash',
             args: [ pgver, pgdata ],
             run: 'never'
+
+        # minio local setup. Use "vagrant up --provision-with=minio"
+        backup.vm.provision 'minio', type: 'shell', 
+          path: 'provision/minio.bash', run: 'never'
     end
 
 end

test/provision/check_pgbackrest.bash

@@ -14,6 +14,7 @@ PACKAGES=(
     perl-JSON
     perl-Net-SFTP-Foreign
     perl-Data-Dumper
+    wget
 )
 
 yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}"

test/provision/icinga2_minio.bash

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# add check_pgbackrest tests
+cat <<EOF >>/etc/icinga2/conf.d/hosts.conf
+
+object CheckCommand "by_ssh_pgbackrest_retention" {
+  import "by_ssh"
+  vars.by_ssh_command = "/usr/lib64/nagios/plugins/check_pgbackrest --stanza=\$stanza$ --service=retention --retention-full=\$retention_full$ --prefix='\$prefix$'"
+}
+
+object Service "pgbackrest_retention" {
+  import "generic-service"
+  host_name = "pgsql-srv"
+  check_command = "by_ssh_pgbackrest_retention"
+  vars.by_ssh_logname = "accessed_by_ssh"
+
+  vars.stanza = "my_stanza"
+  vars.retention_full = 1
+  vars.prefix = "sudo -u postgres"
+}
+
+EOF
+
+systemctl restart icinga2
+
+# show
+icingacli monitoring list services --service=pgbackrest* --verbose

test/provision/minio.bash

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# install MinIO
+# based on https://www.centosblog.com/install-configure-minio-object-storage-server-centos-linux/
+useradd -s /sbin/nologin -d /opt/minio minio
+mkdir -p /opt/minio/bin
+mkdir /opt/minio/data
+
+# download ~40MB, this step will take some time...
+echo "--wget minio: download ~40MB, this step will take some time..."
+wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio -O /opt/minio/bin/minio
+chmod +x /opt/minio/bin/minio
+
+# generate self-signed certificates
+mkdir -p -m 755 /opt/minio/certs
+cd /opt/minio/certs
+openssl genrsa -out ca.key 2048
+openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=BE/ST=Country/L=City/O=Organization/CN=check_pgbackrest"
+openssl genrsa -out server.key 2048
+openssl req -new -key server.key -out server.csr -subj "/C=BE/ST=Country/L=City/O=Organization/CN=check_pgbackrest"
+openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256
+cp /opt/minio/certs/server.crt /opt/minio/certs/public.crt
+cp /opt/minio/certs/server.key /opt/minio/certs/private.key
+chmod 644 /opt/minio/certs/*
+
+# basic configuration
+cat<<EOF >"/opt/minio/minio.conf"
+MINIO_VOLUMES=/opt/minio/data
+MINIO_DOMAIN=minio.local
+MINIO_OPTS="--certs-dir /opt/minio/certs --address :443 --compat"
+MINIO_ACCESS_KEY="accessKey"
+MINIO_SECRET_KEY="superSECRETkey" 
+EOF
+
+cat<<EOF >>"/etc/hosts"
+127.0.0.1 pgbackrest.minio.local minio.local s3.eu-west-3.amazonaws.com
+EOF
+
+chown -R minio:minio /opt/minio
+
+# systemd service
+cat<<EOF >"/etc/systemd/system/minio.service"
+[Unit]
+Description=Minio
+Documentation=https://docs.minio.io
+Wants=network-online.target
+After=network-online.target
+AssertFileIsExecutable=/opt/minio/bin/minio
+
+[Service]
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+WorkingDirectory=/opt/minio
+
+User=minio
+Group=minio
+
+PermissionsStartOnly=true
+
+EnvironmentFile=-/opt/minio/minio.conf
+ExecStartPre=/bin/bash -c "[ -n \\"\${MINIO_VOLUMES}\\" ] || echo \\"Variable MINIO_VOLUMES not set in /opt/minio/minio.conf\\""
+
+ExecStart=/opt/minio/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES
+
+StandardOutput=journal
+StandardError=inherit
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Disable timeout logic and wait until process is stopped
+TimeoutStopSec=0
+
+# SIGTERM signal is used to stop Minio
+KillSignal=SIGTERM
+
+SendSIGKILL=no
+
+SuccessExitStatus=0
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+firewall-cmd --quiet --permanent --add-service=https
+firewall-cmd --quiet --reload
+systemctl enable minio
+systemctl start minio
+sudo -iu postgres psql -c "SELECT pg_sleep(5);" > /dev/null 2>&1
+systemctl status minio
+grep -E 'accessKey|secretKey' /opt/minio/data/.minio.sys/config/config.json
+
+# install s3cmd
+yum --enablerepo epel-testing install --nogpgcheck --quiet -y -e 0 s3cmd
+
+cat<<EOF > ~/.s3cfg
+host_base = minio.local
+host_bucket = pgbackrest.minio.local
+bucket_location = eu-west-3
+use_https = true
+access_key = accessKey
+secret_key = superSECRETkey
+EOF
+
+# create "pgbackrest" bucket and sub-directory "repo1"
+s3cmd mb --no-check-certificate s3://pgbackrest
+mkdir /opt/minio/data/pgbackrest/repo1
+chown minio: /opt/minio/data/pgbackrest/repo1
+s3cmd ls --no-check-certificate s3://pgbackrest/repo1

test/provision/pgbackrest_minio.bash

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+PGVER="$1"
+PGDATA="$2"
+
+PACKAGES=(
+    pgbackrest
+)
+
+yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}"
+
+# pgbackrest.conf setup
+cat<<EOC > "/etc/pgbackrest.conf"
+[global]
+repo1-path=/repo1
+repo1-type=s3
+repo1-s3-endpoint=minio.local
+repo1-s3-bucket=pgbackrest
+repo1-s3-verify-ssl=n
+repo1-s3-key=accessKey
+repo1-s3-key-secret=superSECRETkey
+repo1-s3-region=eu-west-3
+
+repo1-retention-full=1
+process-max=2
+log-level-console=warn
+log-level-file=info
+start-fast=y
+delta=y
+
+[my_stanza]
+pg1-path=${PGDATA}
+EOC
+
+# setup DNS alias for backup-srv where minio is installed
+MINIO_IP=`getent hosts backup-srv | awk '{ print $1 }'`
+cat<<EOF >>"/etc/hosts"
+$MINIO_IP pgbackrest.minio.local minio.local s3.eu-west-3.amazonaws.com
+EOF
+
+# archive_command setup
+sudo -iu postgres pgbackrest --stanza=my_stanza stanza-create
+
+cat <<'EOS' | "/usr/pgsql-${PGVER}/bin/psql" -U postgres
+ALTER SYSTEM SET "archive_command" TO 'pgbackrest --stanza=my_stanza archive-push %p';
+SELECT pg_reload_conf();
+EOS
+
+sudo -iu postgres pgbackrest --stanza=my_stanza check

test/regress/test-s3.bash

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")"
+PLUGIN_PATH=/usr/lib64/nagios/plugins
+RESULTS_DIR=/tmp/results
+
+# Generate the expected results ?
+GENERATE_EXPECTED=false
+
+if $GENERATE_EXPECTED; then
+	RESULTS_DIR=expected
+fi
+
+if [ ! -d $RESULTS_DIR ]; then
+     mkdir $RESULTS_DIR
+fi
+
+## Tests
+# --list
+echo "--list"
+$PLUGIN_PATH/check_pgbackrest --list > $RESULTS_DIR/list.out
+
+# --version
+echo "--version"
+$PLUGIN_PATH/check_pgbackrest --version > $RESULTS_DIR/version.out
+
+# --service=retention missing arg
+echo "--service=retention missing arg"
+$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention > $RESULTS_DIR/retention-missing-arg.out 2>&1
+
+# --service=retention --retention-full
+echo "--service=retention --retention-full"
+sudo -iu postgres pgbackrest --stanza=my_stanza backup --type=full --repo1-retention-full=1
+$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-full=1 | cut -f1 -d"|" > $RESULTS_DIR/retention-full.out
+
+# --service=retention --retention-age
+echo "--service=retention --retention-age"
+$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-age=1h | cut -f1 -d"|" > $RESULTS_DIR/retention-age.out
+
+# --service=retention fail
+echo "--service=retention fail"
+sudo -iu postgres psql -c "SELECT pg_sleep(1);" > /dev/null 2>&1
+$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-full=2 --retention-age=1s | cut -f1 -d"|" > $RESULTS_DIR/retention-fail.out
+
+# # --service=archives missing arg
+# echo "--service=archives missing arg"
+# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives > $RESULTS_DIR/archives-missing-arg.out 2>&1
+
+# # --service=archives --repo-path
+# echo "--service=archives --repo-path"
+# sudo -iu postgres psql -c "SELECT pg_switch_xlog();" > /dev/null 2>&1
+# sudo -iu postgres psql -c "SELECT pg_switch_wal();" > /dev/null 2>&1
+# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive | cut -f1 -d"-" > $RESULTS_DIR/archives-ok.out
+
+# # --service=archives --ignore-archived-since
+# echo "--service=archives --ignore-archived-since"
+# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --ignore-archived-since=1h > $RESULTS_DIR/archives-ignore-since.out
+
+# # --service=archives --latest-archive-age-alert
+# echo "--service=archives --latest-archive-age-alert"
+# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --latest-archive-age-alert=1h | cut -f1 -d"-" > $RESULTS_DIR/archives-age-alert-ok.out
+# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --latest-archive-age-alert=1s | cut -f1 -d"-" > $RESULTS_DIR/archives-age-alert-ko.out
+
+## Results
+diff -abB expected/ $RESULTS_DIR/ > /tmp/regression.diffs
+if [ $(wc -l < /tmp/regression.diffs) -gt 0 ]; then
+     cat /tmp/regression.diffs
+fi