From ad43f4eb5c843f7adbe8c8df50f805fc1b35634e Mon Sep 17 00:00:00 2001 From: Stefan Fercot Date: Thu, 18 Jul 2019 11:57:02 +0200 Subject: [PATCH] Create test scenarios with pgbackrest S3 support, using vagrant and minio --- test/Makefile | 6 ++ test/README.md | 33 ++++++++ test/Vagrantfile | 14 ++++ test/provision/check_pgbackrest.bash | 1 + test/provision/icinga2_minio.bash | 31 ++++++++ test/provision/minio.bash | 112 +++++++++++++++++++++++++++ test/provision/pgbackrest_minio.bash | 53 +++++++++++++ test/regress/test-s3.bash | 68 ++++++++++++++++ 8 files changed, 318 insertions(+) create mode 100644 test/provision/icinga2_minio.bash create mode 100644 test/provision/minio.bash create mode 100644 test/provision/pgbackrest_minio.bash create mode 100755 test/regress/test-s3.bash diff --git a/test/Makefile b/test/Makefile index 3424392..7dc591c 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,5 +1,6 @@ s1: clean create_vm pgbackrest_local s2: clean create_vm pgbackrest_remote +s3: clean create_vm pgbackrest_minio create_vm: vagrant up @@ -13,5 +14,10 @@ pgbackrest_remote: vagrant up --provision-with=pgbackrest_remote_standby vagrant up --provision-with=icinga2_remote +pgbackrest_minio: + vagrant up --provision-with=minio + vagrant up --provision-with=pgbackrest_minio + vagrant up --provision-with=icinga2_minio + clean: vagrant destroy -f \ No newline at end of file diff --git a/test/README.md b/test/README.md index a86240c..edf3627 100644 --- a/test/README.md +++ b/test/README.md @@ -25,6 +25,12 @@ remotely (on `pgsql-srv`). Checks are performed from `icinga-srv` by ssh. pgBackRest backups are use to build a Streaming Replication with `backup-srv` as standby server. +### 3. pgBackRest configured to backup and archive to a MinIO S3 bucket + + * `icinga-srv` executes check_pgbackrest by ssh with Icinga 2; + * `pgsql-srv` hosting a pgsql cluster with pgBackRest installed; + * `backup-srv` hosting the MinIO server. + ## Testing The easiest way to start testing is with the included `Makefile`. @@ -75,6 +81,33 @@ vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s2-from-prima Expected run time: 30 sec. +### Test case 3 + +_Build_: + +```bash +cd test +make s3 +``` + +Expected make time: 7 min. + +_Check the results of a manual execution of check_pgbackrest_: + +```bash +vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/test-s3.bash" +``` + +Expected run time: 40 sec. + +_To simulate some activity with pgBackRest_: + +```bash +vagrant ssh pgsql-srv -c "sudo /check_pgbackrest/test/regress/simulate-activity-local.bash" +``` + +Modify `SCALE` factor or `ACTIVITY_TIME` in this script to simulate more activity. + ### Icinga 2 connectivity _Check the results of _check_pgbackrest_ launched by Icinga 2: diff --git a/test/Vagrantfile b/test/Vagrantfile index 731e88d..f16d937 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -50,6 +50,10 @@ Vagrant.configure(2) do |config| # pgbackrest remote setup. Use "vagrant up --provision-with=icinga2_remote" icinga.vm.provision 'icinga2_remote', type: 'shell', path: 'provision/icinga2_remote.bash', run: 'never' + + # pgbackrest minio setup. Use "vagrant up --provision-with=icinga2_minio" + icinga.vm.provision 'icinga2_minio', type: 'shell', + path: 'provision/icinga2_minio.bash', run: 'never' end config.vm.define "pgsql-srv" do |pgsql| @@ -66,6 +70,12 @@ Vagrant.configure(2) do |config| path: 'provision/pgbackrest_remote_primary.bash', args: [ pgver, pgdata ], run: 'never' + + # pgbackrest minio setup. Use "vagrant up --provision-with=pgbackrest_minio" + pgsql.vm.provision 'pgbackrest_minio', type: 'shell', + path: 'provision/pgbackrest_minio.bash', + args: [ pgver, pgdata ], + run: 'never' end config.vm.define "backup-srv" do |backup| @@ -76,6 +86,10 @@ Vagrant.configure(2) do |config| path: 'provision/pgbackrest_remote_standby.bash', args: [ pgver, pgdata ], run: 'never' + + # minio local setup. Use "vagrant up --provision-with=minio" + backup.vm.provision 'minio', type: 'shell', + path: 'provision/minio.bash', run: 'never' end end \ No newline at end of file diff --git a/test/provision/check_pgbackrest.bash b/test/provision/check_pgbackrest.bash index 6ff9612..d1fe125 100644 --- a/test/provision/check_pgbackrest.bash +++ b/test/provision/check_pgbackrest.bash @@ -14,6 +14,7 @@ PACKAGES=( perl-JSON perl-Net-SFTP-Foreign perl-Data-Dumper + wget ) yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}" diff --git a/test/provision/icinga2_minio.bash b/test/provision/icinga2_minio.bash new file mode 100644 index 0000000..688b8c9 --- /dev/null +++ b/test/provision/icinga2_minio.bash @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# add check_pgbackrest tests +cat <>/etc/icinga2/conf.d/hosts.conf + +object CheckCommand "by_ssh_pgbackrest_retention" { + import "by_ssh" + vars.by_ssh_command = "/usr/lib64/nagios/plugins/check_pgbackrest --stanza=\$stanza$ --service=retention --retention-full=\$retention_full$ --prefix='\$prefix$'" +} + +object Service "pgbackrest_retention" { + import "generic-service" + host_name = "pgsql-srv" + check_command = "by_ssh_pgbackrest_retention" + vars.by_ssh_logname = "accessed_by_ssh" + + vars.stanza = "my_stanza" + vars.retention_full = 1 + vars.prefix = "sudo -u postgres" +} + +EOF + +systemctl restart icinga2 + +# show +icingacli monitoring list services --service=pgbackrest* --verbose \ No newline at end of file diff --git a/test/provision/minio.bash b/test/provision/minio.bash new file mode 100644 index 0000000..d3ff2e7 --- /dev/null +++ b/test/provision/minio.bash @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# install MinIO +# based on https://www.centosblog.com/install-configure-minio-object-storage-server-centos-linux/ +useradd -s /sbin/nologin -d /opt/minio minio +mkdir -p /opt/minio/bin +mkdir /opt/minio/data + +# download ~40MB, this step will take some time... +echo "--wget minio: download ~40MB, this step will take some time..." +wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio -O /opt/minio/bin/minio +chmod +x /opt/minio/bin/minio + +# generate self-signed certificates +mkdir -p -m 755 /opt/minio/certs +cd /opt/minio/certs +openssl genrsa -out ca.key 2048 +openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=BE/ST=Country/L=City/O=Organization/CN=check_pgbackrest" +openssl genrsa -out server.key 2048 +openssl req -new -key server.key -out server.csr -subj "/C=BE/ST=Country/L=City/O=Organization/CN=check_pgbackrest" +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 -sha256 +cp /opt/minio/certs/server.crt /opt/minio/certs/public.crt +cp /opt/minio/certs/server.key /opt/minio/certs/private.key +chmod 644 /opt/minio/certs/* + +# basic configuration +cat<"/opt/minio/minio.conf" +MINIO_VOLUMES=/opt/minio/data +MINIO_DOMAIN=minio.local +MINIO_OPTS="--certs-dir /opt/minio/certs --address :443 --compat" +MINIO_ACCESS_KEY="accessKey" +MINIO_SECRET_KEY="superSECRETkey" +EOF + +cat<>"/etc/hosts" +127.0.0.1 pgbackrest.minio.local minio.local s3.eu-west-3.amazonaws.com +EOF + +chown -R minio:minio /opt/minio + +# systemd service +cat<"/etc/systemd/system/minio.service" +[Unit] +Description=Minio +Documentation=https://docs.minio.io +Wants=network-online.target +After=network-online.target +AssertFileIsExecutable=/opt/minio/bin/minio + +[Service] +AmbientCapabilities=CAP_NET_BIND_SERVICE +WorkingDirectory=/opt/minio + +User=minio +Group=minio + +PermissionsStartOnly=true + +EnvironmentFile=-/opt/minio/minio.conf +ExecStartPre=/bin/bash -c "[ -n \\"\${MINIO_VOLUMES}\\" ] || echo \\"Variable MINIO_VOLUMES not set in /opt/minio/minio.conf\\"" + +ExecStart=/opt/minio/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES + +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop Minio +KillSignal=SIGTERM + +SendSIGKILL=no + +SuccessExitStatus=0 + +[Install] +WantedBy=multi-user.target +EOF + +firewall-cmd --quiet --permanent --add-service=https +firewall-cmd --quiet --reload +systemctl enable minio +systemctl start minio +sudo -iu postgres psql -c "SELECT pg_sleep(5);" > /dev/null 2>&1 +systemctl status minio +grep -E 'accessKey|secretKey' /opt/minio/data/.minio.sys/config/config.json + +# install s3cmd +yum --enablerepo epel-testing install --nogpgcheck --quiet -y -e 0 s3cmd + +cat< ~/.s3cfg +host_base = minio.local +host_bucket = pgbackrest.minio.local +bucket_location = eu-west-3 +use_https = true +access_key = accessKey +secret_key = superSECRETkey +EOF + +# create "pgbackrest" bucket and sub-directory "repo1" +s3cmd mb --no-check-certificate s3://pgbackrest +mkdir /opt/minio/data/pgbackrest/repo1 +chown minio: /opt/minio/data/pgbackrest/repo1 +s3cmd ls --no-check-certificate s3://pgbackrest/repo1 \ No newline at end of file diff --git a/test/provision/pgbackrest_minio.bash b/test/provision/pgbackrest_minio.bash new file mode 100644 index 0000000..4b54beb --- /dev/null +++ b/test/provision/pgbackrest_minio.bash @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +PGDATA="$2" + +PACKAGES=( + pgbackrest +) + +yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}" + +# pgbackrest.conf setup +cat< "/etc/pgbackrest.conf" +[global] +repo1-path=/repo1 +repo1-type=s3 +repo1-s3-endpoint=minio.local +repo1-s3-bucket=pgbackrest +repo1-s3-verify-ssl=n +repo1-s3-key=accessKey +repo1-s3-key-secret=superSECRETkey +repo1-s3-region=eu-west-3 + +repo1-retention-full=1 +process-max=2 +log-level-console=warn +log-level-file=info +start-fast=y +delta=y + +[my_stanza] +pg1-path=${PGDATA} +EOC + +# setup DNS alias for backup-srv where minio is installed +MINIO_IP=`getent hosts backup-srv | awk '{ print $1 }'` +cat<>"/etc/hosts" +$MINIO_IP pgbackrest.minio.local minio.local s3.eu-west-3.amazonaws.com +EOF + +# archive_command setup +sudo -iu postgres pgbackrest --stanza=my_stanza stanza-create + +cat <<'EOS' | "/usr/pgsql-${PGVER}/bin/psql" -U postgres +ALTER SYSTEM SET "archive_command" TO 'pgbackrest --stanza=my_stanza archive-push %p'; +SELECT pg_reload_conf(); +EOS + +sudo -iu postgres pgbackrest --stanza=my_stanza check \ No newline at end of file diff --git a/test/regress/test-s3.bash b/test/regress/test-s3.bash new file mode 100755 index 0000000..8f752a9 --- /dev/null +++ b/test/regress/test-s3.bash @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" +PLUGIN_PATH=/usr/lib64/nagios/plugins +RESULTS_DIR=/tmp/results + +# Generate the expected results ? +GENERATE_EXPECTED=false + +if $GENERATE_EXPECTED; then + RESULTS_DIR=expected +fi + +if [ ! -d $RESULTS_DIR ]; then + mkdir $RESULTS_DIR +fi + +## Tests +# --list +echo "--list" +$PLUGIN_PATH/check_pgbackrest --list > $RESULTS_DIR/list.out + +# --version +echo "--version" +$PLUGIN_PATH/check_pgbackrest --version > $RESULTS_DIR/version.out + +# --service=retention missing arg +echo "--service=retention missing arg" +$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention > $RESULTS_DIR/retention-missing-arg.out 2>&1 + +# --service=retention --retention-full +echo "--service=retention --retention-full" +sudo -iu postgres pgbackrest --stanza=my_stanza backup --type=full --repo1-retention-full=1 +$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-full=1 | cut -f1 -d"|" > $RESULTS_DIR/retention-full.out + +# --service=retention --retention-age +echo "--service=retention --retention-age" +$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-age=1h | cut -f1 -d"|" > $RESULTS_DIR/retention-age.out + +# --service=retention fail +echo "--service=retention fail" +sudo -iu postgres psql -c "SELECT pg_sleep(1);" > /dev/null 2>&1 +$PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=retention --retention-full=2 --retention-age=1s | cut -f1 -d"|" > $RESULTS_DIR/retention-fail.out + +# # --service=archives missing arg +# echo "--service=archives missing arg" +# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives > $RESULTS_DIR/archives-missing-arg.out 2>&1 + +# # --service=archives --repo-path +# echo "--service=archives --repo-path" +# sudo -iu postgres psql -c "SELECT pg_switch_xlog();" > /dev/null 2>&1 +# sudo -iu postgres psql -c "SELECT pg_switch_wal();" > /dev/null 2>&1 +# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive | cut -f1 -d"-" > $RESULTS_DIR/archives-ok.out + +# # --service=archives --ignore-archived-since +# echo "--service=archives --ignore-archived-since" +# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --ignore-archived-since=1h > $RESULTS_DIR/archives-ignore-since.out + +# # --service=archives --latest-archive-age-alert +# echo "--service=archives --latest-archive-age-alert" +# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --latest-archive-age-alert=1h | cut -f1 -d"-" > $RESULTS_DIR/archives-age-alert-ok.out +# $PLUGIN_PATH/check_pgbackrest --stanza=my_stanza --service=archives --repo-path=/var/lib/pgbackrest/archive --latest-archive-age-alert=1s | cut -f1 -d"-" > $RESULTS_DIR/archives-age-alert-ko.out + +## Results +diff -abB expected/ $RESULTS_DIR/ > /tmp/regression.diffs +if [ $(wc -l < /tmp/regression.diffs) -gt 0 ]; then + cat /tmp/regression.diffs +fi