Add basic support for pgbackrest multi-repo feature (#22)

* add basic support for pgbackrest multi-repo feature
* new test suite using ansible
* add check_pgbackrest regress tests to the new test suite

.github/workflows/main.yml

@@ -0,0 +1,75 @@
+---
+name: main
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  use-case-1:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - DOCKERI: centos:7
+            DBTYPE: PG
+            DBVERSION: 13
+            CLNAME: c7pg
+            EXTRA_VARS: "pgbackrest_excpected_release=2.34 check_pgbackrest_build=true"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: shogo82148/actions-setup-perl@v1
+
+      - name: Initial step
+        run: sh run.sh -i
+
+      - name: Run CI script
+        env:
+          EDB_REPO_USERNAME: ${{ secrets.EDB_REPO_USERNAME }}
+          EDB_REPO_PASSWORD: ${{ secrets.EDB_REPO_PASSWORD }}
+          ARCH: use-case-1
+          CLPATH: /home/runner/clusters
+          CLNAME: ${{ matrix.CLNAME }}
+          DBTYPE: ${{ matrix.DBTYPE }}
+          DBVERSION: ${{ matrix.DBVERSION }}
+          DOCKERI: ${{ matrix.DOCKERI }}
+          EXTRA_VARS: ${{ matrix.EXTRA_VARS }}
+        run: sh ci.sh
+
+  use-case-2:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - DOCKERI: ubuntu:20.04
+            DBTYPE: PG
+            DBVERSION: 13
+            CLNAME: u20pg
+            EXTRA_VARS: "pgbackrest_excpected_release=2.34 check_pgbackrest_build=true"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: shogo82148/actions-setup-perl@v1
+
+      - name: Initial step
+        run: sh run.sh -i
+
+      - name: Run CI script
+        env:
+          EDB_REPO_USERNAME: ${{ secrets.EDB_REPO_USERNAME }}
+          EDB_REPO_PASSWORD: ${{ secrets.EDB_REPO_PASSWORD }}
+          ARCH: use-case-2
+          CLPATH: /home/runner/clusters
+          CLNAME: ${{ matrix.CLNAME }}
+          DBTYPE: ${{ matrix.DBTYPE }}
+          DBVERSION: ${{ matrix.DBVERSION }}
+          DOCKERI: ${{ matrix.DOCKERI }}
+          EXTRA_VARS: ${{ matrix.EXTRA_VARS }}
+        run: sh ci.sh

.gitignore

@@ -1,6 +1,4 @@
 .vagrant/
-tests/common/perf/nytprof*
 vagrant.yml
-tests/common/configuration.profile
-_old.*
-*.rpm
+tests/validation.sh
+tests/validation.log

CHANGELOG.md

@@ -3,6 +3,10 @@ Changelog
 
 2021-xx-xx v2.1:
 
+  - Only support pgBackRest **2.33** and above in order to introduce some basic
+  support for the multi-repository feature.
+  Introduce the `--repo` option to set the repository index to operate on.
+  When multiple repositories will be found, the `--repo` option is mandatory.
   - ...
 
 

README

@@ -18,6 +18,10 @@ DESCRIPTION
     -S, --stanza NAME
         Name of the stanza to check.
 
+    --repo REPOSITORY
+        Repository index to operate on. The "--repo" argument is mandatory
+        when multiple repositories are defined.
+
     -O, --output OUTPUT_FORMAT
         The output format. Supported outputs are: "human", "json" and
         "nagios" (default).

README.pod

@@ -24,6 +24,11 @@ list.
 
 Name of the stanza to check.
 
+=item B<--repo> REPOSITORY
+
+Repository index to operate on. The C<--repo> argument is mandatory when 
+multiple repositories are defined.
+
 =item B<-O>, B<--output> OUTPUT_FORMAT
 
 The output format. Supported outputs are: C<human>, C<json> and C<nagios> (default).

check_pgbackrest

@@ -63,7 +63,7 @@ my $output_fmt;
 
 $VERSION = '2.1dev';
 $PROGRAM = 'check_pgbackrest';
-$PGBR_SUPPORT = '2.32';
+$PGBR_SUPPORT = '2.33';
 $INIT_TIME = time();
 
 # Available services and descriptions.
@@ -99,6 +99,11 @@ list.
 
 Name of the stanza to check.
 
+=item B<--repo> REPOSITORY
+
+Repository index to operate on. The C<--repo> argument is mandatory when 
+multiple repositories are defined.
+
 =item B<-O>, B<--output> OUTPUT_FORMAT
 
 The output format. Supported outputs are: C<human>, C<json> and C<nagios> (default).
@@ -427,7 +432,11 @@ sub pgbackrest_info {
 
     if(defined $args{'config'}) {
         $infocmd .= " --config=".$args{'config'};
-    }    
+    }
+
+    if(defined $args{'repo'}) {
+        $infocmd .= " --repo=".$args{'repo'};
+    }
 
     if(defined $args{'prefix'}) {
         $infocmd = $args{'prefix'}." $infocmd";
@@ -467,6 +476,10 @@ sub pgbackrest_get {
         $getcmd .= " --config=".$args{'config'};
     }
 
+    if(defined $args{'repo'}) {
+        $getcmd .= " --repo=".$args{'repo'};
+    }
+
     if(defined $args{'prefix'}) {
         $getcmd = $args{'prefix'}." $getcmd";
     }
@@ -501,7 +514,11 @@ sub pgbackrest_ls {
 
     if(defined $args{'config'}) {
         $lscmd .= " --config=".$args{'config'};
-    }    
+    }
+
+    if(defined $args{'repo'}) {
+        $lscmd .= " --repo=".$args{'repo'};
+    }
 
     if(defined $args{'prefix'}) {
         $lscmd = $args{'prefix'}." $lscmd";
@@ -576,6 +593,12 @@ sub check_retention {
     die("Can't get pgBackRest info.\n") unless (defined $backups_info);
 
     if($backups_info->{'status'}->{'code'} == 0) {
+        # Require the --repo option if multiple repositories are defined
+        pod2usage(
+            -message => 'FATAL: multiple repositories found, you must provide --repo.',
+            -exitval => 127
+        ) if scalar(@{$backups_info->{'repo'}}) > 1 and not defined $args{'repo'};
+
         my @full_bck;
         my @diff_bck;
         my @incr_bck;
@@ -629,6 +652,13 @@ sub check_retention {
         }
     }else{
         push @crit_msg, $backups_info->{'status'}->{'message'};
+
+        # Add detailed messages in case of multiple repositories
+        if(scalar(@{$backups_info->{'repo'}}) > 1) {
+            foreach my $line (@{$backups_info->{'repo'}}) {
+                push @crit_msg, "repo".$line->{'key'}." - ".$line->{'status'}->{'message'};
+            }
+        }
     }
 
     return critical($me, \@crit_msg, \@longmsg) if @crit_msg;
@@ -734,7 +764,7 @@ sub get_archived_wal_list {
             push @filelist, [substr($filename, 0, 24), $filename, $list->{$key}->{'time'}, $list->{$key}->{'size'}, "$archives_dir/$key"];
 
         }elsif($filename =~ /$history_re_full/ && $start_tl ne $end_tl){
-            # Look for the last history file if needed
+            # Look for the last history file if needed
             dprint("history file to open : $archives_dir/$key\n");
 
             my $history_content = pgbackrest_get(\%args, $archives_dir, $filename);
@@ -807,6 +837,12 @@ sub check_wal_archives {
     dprint("!> pgBackRest info took ".(time() - $start_time)."s\n");
 
     if($backups_info->{'status'}->{'code'} == 0) {
+        # Require the --repo option if multiple repositories are defined
+        pod2usage(
+            -message => 'FATAL: multiple repositories found, you must provide --repo.',
+            -exitval => 127
+        ) if scalar(@{$backups_info->{'repo'}}) > 1 and not defined $args{'repo'};
+
         my $archives_dir = "archive/".$args{'stanza'}."/".$backups_info->{'archive'}[0]->{'id'}; # Relative path inside repository
         dprint("archives_dir: $archives_dir\n");
         my $min_wal = $backups_info->{'archive'}[0]->{'min'};
@@ -1045,6 +1081,7 @@ GetOptions(
         'list-archives|L!',
         'output|O=s',
         'prefix|P=s',
+        'repo=s',
         'retention-age=s',
         'retention-age-to-full=s',
         'retention-full=i',

tests/Makefile

@@ -0,0 +1,42 @@
+### Global variables
+# disable running the activity script by default
+ACTIVITY='false'
+# define PG on CentOS 7 as default profile
+PROFILE='c7pg'
+
+### Make targets
+init: clean_vm create_vm
+
+uc1:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-1' EXTRA="$(EXTRA) check_pgbackrest_build=true deploy_icinga2=true" PGBR_BUILD='false' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+uc1_full:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-1' EXTRA="$(EXTRA) check_pgbackrest_build=true deploy_icinga2=true" PGBR_BUILD='true' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+uc1_light:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-1' EXTRA="$(EXTRA)" PGBR_BUILD='false' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+
+uc2:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-2' EXTRA="$(EXTRA) check_pgbackrest_build=true deploy_icinga2=true" PGBR_BUILD='false' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+uc2_full:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-2' EXTRA="$(EXTRA) check_pgbackrest_build=true deploy_icinga2=true" PGBR_BUILD='true' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+uc2_light:
+	ACTIVITY=$(ACTIVITY) ARCH='use-case-2' EXTRA="$(EXTRA)" PGBR_BUILD='false' PGBR_REPO_TYPE=$(PGBR_REPO_TYPE) PROFILE=$(PROFILE) vagrant up --provision-with=exec-ci
+
+### Setup
+clean_ci:
+	PROFILE=$(PROFILE) vagrant up --provision-with=clean-ci
+
+clean_docker:
+	vagrant ssh -c "docker rm -f $(docker ps -a -q)"
+
+clean_git:
+	git clean -f -dX --dry-run
+
+clean_icinga2:
+	vagrant ssh -c "docker stop $(PROFILE)-icinga2 && docker rm $(PROFILE)-icinga2"
+
+clean_vm:
+	vagrant destroy -f
+
+create_vm:
+	vagrant up
+	vagrant ssh -c "sh /vagrant/run.sh -i"

tests/README.md

@@ -0,0 +1,79 @@
+# README
+
+<!--
+(\_/)
+( •_•)
+/ > 🐘
+-->
+
+---
+
+## Introduction
+
+This _Test Suite_ is based on the [edb-ansible](https://github.com/EnterpriseDB/edb-ansible) Ansible Collection. It deploys docker containers and typical architectures.
+
+It tends to support the following situations:
+* Directly-attached storage - [Use Case 1](https://www.enterprisedb.com/docs/supported-open-source/pgbackrest/06-use_case_1/)
+* Dedicated repository host - [Use Case 2](https://www.enterprisedb.com/docs/supported-open-source/pgbackrest/07-use_case_2)
+
+---
+
+## GitHub Actions
+
+[GitHub Actions](../.github/workflows/main.yml) are testing:
+  * Use-Case 1: PG 13, CentOS 7, using pgBackRest PGDG packages
+  * Use-Case 2: PG 13, Ubuntu 20.04, using pgBackRest PGDG packages
+
+---
+
+## Vagrant
+
+To be able to run the tests manually, define your EDB repositories personal credential `vagrant.yml`. Example in [vagrant.yml-dist](vagrant.yml-dist).
+
+First of all, initialize the virtual machine with `make init`.
+
+* Deploy Use-Case 1 and run the activity script: `make ACTIVITY=true uc1`
+* Deploy Use-Case 2 and run the activity script: `make ACTIVITY=true uc2`
+
+To build pgBackRest from sources, use `uc1_full` or `uc2_full` make targets.
+
+To install pgBackRest and **check_pgbackrest** using PGDG packages, without deploying Icinga2, use `uc1_light` or `uc2_light` make targets.
+
+### Change the test profile
+
+Add `PROFILE=xxx` to the make command.
+
+Available profiles: `c7epas`, `c7pg`, `d10epas`, `d10pg`, `u20epas`, `u20pg`.
+
+### Change the pgBackRest repository type
+
+Add `PGBR_REPO_TYPE=xxx` to the make command.
+
+Available types: `azure`, `s3`, `multi`, `posix`.
+
+When setting `multi` repository, both `s3` and `azure` will be used. When setting `posix` repository, the repository path will be automatically adjusted to `/shared/repo1` where */shared* is a shared volume between the docker containers.
+
+### Icinga2
+
+To interact with Icinga2, the easiest way is to use the API:
+
+```bash
+# Login to the vagrant box
+$ vagrant ssh
+
+# Reschedule check_pgbackrest checks
+$ curl -k -s -u 'icinga2-director:anyPassWord' -H 'Accept: application/json' -X POST \
+ 'https://localhost:5665/v1/actions/reschedule-check' \
+ -d '{ "type": "Service", "filter": "match(pattern,service.name)", "filter_vars": { "pattern": "pgbackrest*" }, "pretty": true }' |jq
+
+# Get check_pgbackrest checks status
+$ curl -k -s -u 'icinga2-director:anyPassWord' -H 'Accept: application/json' -X GET \
+ 'https://localhost:5665/v1/objects/services' \
+ -d '{ "filter": "match(pattern,service.name)", "filter_vars": { "pattern": "pgbackrest*" } }' |jq
+```
+
+### Cleaning
+
+Before changing the `PROFILE` to deploy a new architecture, remove the docker containers and cluster directory using `make PROFILE=xxx clean_ci`.
+
+To remove the vagrant virtual machine: `make clean_vm`.