Add max-archives-check-number option for the archives service

CHANGELOG.md

@@ -7,6 +7,9 @@ Changelog
   support for the multi-repository feature.
   Introduce the `--repo` option to set the repository index to operate on.
   When multiple repositories will be found, the `--repo` option is mandatory.
+  - Add a new `max-archives-check-number` option for the archives service.
+  This is intended to use in case of timeline switch and when boundary WAL can't
+  be detected properly, in order to prevent infinite WAL archives check.
   - ...
 
 

README

@@ -110,6 +110,12 @@ DESCRIPTION
         Use the "--extended-check" argument to force a full check of the
         found archives and raise warnings in case of inconsistencies.
 
+        When WAL archives on different timelines are found, .history files
+        are parsed to find the switch point and define the boundary WAL.
+
+        Use the "--max-archives-check-number" to prevent infinite WAL
+        archives check when boundary WAL can't be defined properly.
+
     check_pgb_version
         Check if this script is running a given version.
 

README.pod

@@ -131,6 +131,12 @@ are ignored.
 Use the C<--extended-check> argument to force a full check of the found 
 archives and raise warnings in case of inconsistencies.
 
+When WAL archives on different timelines are found, .history files are parsed to
+find the switch point and define the boundary WAL.
+
+Use the C<--max-archives-check-number> to prevent infinite WAL archives check
+when boundary WAL can't be defined properly.
+
 =item B<check_pgb_version>
 
 Check if this script is running a given version.

check_pgbackrest

@@ -711,6 +711,12 @@ are ignored.
 Use the C<--extended-check> argument to force a full check of the found 
 archives and raise warnings in case of inconsistencies.
 
+When WAL archives on different timelines are found, .history files are parsed to
+find the switch point and define the boundary WAL.
+
+Use the C<--max-archives-check-number> to prevent infinite WAL archives check
+when boundary WAL can't be defined properly.
+
 =cut
 
 sub get_archived_wal_list {
@@ -794,6 +800,8 @@ sub generate_needed_wal_archives_list {
     my $timeline = hex($start_tl);
     my $wal = hex(substr($min_wal, 8, 8));
     my $seg = hex(substr($min_wal, 16, 8));
+    my $args_ref = shift;
+    my %args = %{ $args_ref };
 
     # Generate list
     my $curr = $min_wal;
@@ -816,6 +824,13 @@ sub generate_needed_wal_archives_list {
         }else{
             push @needed_wal_archives_list, $curr;
         }
+
+        # Break the loop in case max-archives-check-number is defined
+        # Infinite loop might happen when there's a timeline switch but boundary WAL isn't detected correctly
+        die("max-archives-check-number limit exceeded.\n") if (
+            defined $args{'max-archives-check-number'} and
+            scalar(@needed_wal_archives_list) > $args{'max-archives-check-number'}
+        );
     }
 
     my @unique_needed_wal_archives_list = do { my %seen; grep { !$seen{$_}++ } @needed_wal_archives_list };
@@ -872,7 +887,7 @@ sub check_wal_archives {
         # Sort by filename
         my @filelist_sorted = sort { $a->[0] cmp $b->[0] }
             grep{ (defined($_->[0]) and defined($_->[1]))
-                or die "Can't read WAL files."
+                or die "Can't read WAL files.\n"
             } @filelist;
 
         my @filelist_simplified;
@@ -920,7 +935,7 @@ sub check_wal_archives {
         $seg_per_wal-- if $dbver <= 92;
         dprint("Get all the needed WAL archives...\n");
         $start_time = time();
-        my @needed_wal_archives_list=&generate_needed_wal_archives_list($min_wal, $max_wal, \@branch_wals, $seg_per_wal);
+        my @needed_wal_archives_list=&generate_needed_wal_archives_list($min_wal, $max_wal, \@branch_wals, $seg_per_wal, \%args);
         dprint("!> Get all the needed WAL archives took ".(time() - $start_time)."s\n");
 
         # Get the latest backup info
@@ -971,7 +986,7 @@ sub check_wal_archives {
                 next;
             }
 
-            foreach my $needed_wal (&generate_needed_wal_archives_list($line->{'archive'}->{'start'}, $line->{'archive'}->{'stop'}, \@branch_wals, $seg_per_wal)) {
+            foreach my $needed_wal (&generate_needed_wal_archives_list($line->{'archive'}->{'start'}, $line->{'archive'}->{'stop'}, \@branch_wals, $seg_per_wal, \%args)) {
                 unless ( $filelist_simplified_hash{ $needed_wal } ) {
                     push @crit_missing_files => $needed_wal;
                 }
@@ -1079,6 +1094,7 @@ GetOptions(
         'latest-archive-age-alert=s',
         'list|l!',
         'list-archives|L!',
+        'max-archives-check-number=s',
         'output|O=s',
         'prefix|P=s',
         'repo=s',
@@ -1118,11 +1134,11 @@ pod2usage(
 ) if ( $args{'retention-age'} or $args{'retention-age-to-full'} or $args{'retention-full'} )
     and $args{'service'} ne 'retention';
 
-# Check "archives" specific args --extended-check, --ignore-archived-after, --ignore-archived-before and --latest-archive-age-alert
+# Check "archives" specific args --extended-check, --ignore-archived-after, --ignore-archived-before, --latest-archive-age-alert and --max-archives-check-number
 pod2usage(
-    -message => 'FATAL: "extended-check", "ignore-archived-after", "ignore-archived-before" and "latest-archive-age-alert" are only allowed with "archives" service.',
+    -message => 'FATAL: "extended-check", "ignore-archived-after", "ignore-archived-before", "latest-archive-age-alert" and "max-archives-check-number" are only allowed with "archives" service.',
     -exitval => 127
-) if ( $args{'extended-check'} or $args{'ignore-archived-after'} or $args{'ignore-archived-before'} or $args{'latest-archive-age-alert'} )
+) if ( $args{'extended-check'} or $args{'ignore-archived-after'} or $args{'ignore-archived-before'} or $args{'latest-archive-age-alert'} or $args{'max-archives-check-number'} )
     and $args{'service'} ne 'archives';
 
 # Check "archives" specific arg --list-archives

tests/VALIDATION.md

@@ -48,4 +48,4 @@ time make ACTIVITY=true PROFILE=u20epas PGBR_REPO_TYPE=multi uc2
 make PROFILE=u20epas clean_ci
 ```
 
-* To build pgBackRest from sources, use `uc1_full` of `uc2_full` make target
+* To build pgBackRest from sources, use `uc1_full` of `uc2_full` make target

tests/playbooks/regress/expected/archives-max-archives-check-ko.out

@@ -0,0 +1 @@
+max-archives-check-number limit exceeded.

tests/playbooks/regress/regression-tests.bash

@@ -61,6 +61,11 @@ if [ ! -z "$PGBR_HOST" ]; then
     PGBR_HOST=(`$PYTHON -c "print(' '.join($PGBR_HOST))"`)
 fi
 echo "PGBR_REPO_TYPE = $PGBR_REPO_TYPE"
+REPO=""
+if [ "$PGBR_REPO_TYPE" = "multi" ]; then
+    REPO="--repo=1"
+    echo "...multi repo support, defaulting to repo1"
+fi
 
 if [ ! -d $RESULTS_DIR ]; then
      mkdir $RESULTS_DIR
@@ -71,13 +76,13 @@ fi
 if ! $SKIP_INIT; then
     echo "...Initiate backups (full, diff, incr)"
     if [ "$SCRIPT_PROFILE" = "local" ]; then
-        sudo -iu $PGUSER pgbackrest --stanza=$STANZA backup --type=full --repo1-retention-full=1
-        sudo -iu $PGUSER pgbackrest --stanza=$STANZA backup --type=diff
-        sudo -iu $PGUSER pgbackrest --stanza=$STANZA backup --type=incr
+        sudo -iu $PGUSER pgbackrest --stanza=$STANZA $REPO backup --type=full --repo1-retention-full=1
+        sudo -iu $PGUSER pgbackrest --stanza=$STANZA $REPO backup --type=diff
+        sudo -iu $PGUSER pgbackrest --stanza=$STANZA $REPO backup --type=incr
     else
-        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA backup --type=full --repo1-retention-full=1"
-        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA backup --type=diff"
-        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA backup --type=incr"
+        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA $REPO backup --type=full --repo1-retention-full=1"
+        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA $REPO backup --type=diff"
+        sudo -iu $PGUSER ssh ${SSH_ARGS} ${PGBR_USER}@${PGBR_HOST} "pgbackrest --stanza=$STANZA $REPO backup --type=incr"
     fi
 fi
 
@@ -89,11 +94,6 @@ $PLUGIN_PATH/check_pgbackrest --list | tee $RESULTS_DIR/list.out
 echo "--version"
 $PLUGIN_PATH/check_pgbackrest --version
 
-REPO=""
-if [ "$PGBR_REPO_TYPE" = "multi" ]; then
-    REPO="--repo=1"
-    echo "...multi repo support, defaulting to repo1"
-fi
 
 # --service=retention --retention-full
 echo "--service=retention --retention-full"
@@ -118,6 +118,7 @@ $PLUGIN_PATH/check_pgbackrest --prefix="sudo -u $PGUSER" --stanza=$STANZA $REPO
 
 # --service=archives
 echo "--service=archives"
+sudo -iu $PGUSER $PGBIN/psql -h $PGUNIXSOCKET -d $PGDATABASE -c "SELECT pg_create_restore_point('generate WAL');" > /dev/null 2>&1
 sudo -iu $PGUSER $PGBIN/psql -h $PGUNIXSOCKET -d $PGDATABASE -c "SELECT pg_switch_xlog();" > /dev/null 2>&1
 sudo -iu $PGUSER $PGBIN/psql -h $PGUNIXSOCKET -d $PGDATABASE -c "SELECT pg_switch_wal();" > /dev/null 2>&1
 sudo -iu $PGUSER $PGBIN/psql -h $PGUNIXSOCKET -d $PGDATABASE -c "SELECT pg_sleep(1);" > /dev/null 2>&1
@@ -140,6 +141,10 @@ $PLUGIN_PATH/check_pgbackrest --prefix="sudo -u $PGUSER" --stanza=$STANZA $REPO
 $PLUGIN_PATH/check_pgbackrest --prefix="sudo -u $PGUSER" --stanza=$STANZA $REPO --service=archives --latest-archive-age-alert=1s --output=human
 $PLUGIN_PATH/check_pgbackrest --prefix="sudo -u $PGUSER" --stanza=$STANZA $REPO --service=archives --latest-archive-age-alert=1s | cut -f1 -d"-" > $RESULTS_DIR/archives-age-alert-ko.out
 
+# --service=archives --max-archives-check-number
+echo "--service=archives --max-archives-check-number"
+$PLUGIN_PATH/check_pgbackrest --prefix="sudo -u $PGUSER" --stanza=$STANZA $REPO --service=archives --max-archives-check-number=1 > $RESULTS_DIR/archives-max-archives-check-ko.out 2>&1
+
 ## Results
 diff -abB expected/ $RESULTS_DIR/ > /tmp/regression.diffs
 if [ $(wc -l < /tmp/regression.diffs) -gt 0 ]; then