borgsetup

#!/bin/bash

SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
. "${SCRIPT_DIR}/poshlib/poshlib.sh" || exit 1
use strict
use utils
use parse-opt

YQ=/usr/local/bin/yq

# Number of alphanumeric chars in encryption passphrase.
# 24 should be good enough
PASSPHRASE_LENGTH=24

# Uncomment to perform an initial backup straight away
#IMMEDIATE_BACKUP=true

# Where to keep our borgmatic-specific ssh key
SSH_KEYFILE=/root/.ssh/id_rsa_borgmatic

# Configure a location under which we keep backup repositories.
# Machine-specific repositories are created under this with automatic names.
# Locations SHOULD be empty apart from borg backup repositories.

# If we are using a remote location, configure it here. It MUST already exist.
# The remote directory should usually NOT start with a /, and can be empty.
REMOTE_USER=user
REMOTE_HOST=host.example.net
REMOTE_DIRECTORY=borg
REMOTE_BORG_PATH=borg
# Supplying a known_hosts keyline prevents us being asked for confirmation
# when connecting to the remote host for the first time, which can cause
# ansible to hang.
#REMOTE_HOST_KEY="host.example.net,10.0.0.1 ssh-ed25519 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"

# List of directories to back up
BACKUP_SOURCES=( /data /etc /home /opt /root /srv /usr/local /var )

# Where to search for borgmatic
BORGMATIC=/usr/bin/borgmatic
ADDITIONAL_PATH=""

# If we are using a local directory, configure it here.
#LOCAL_DIRECTORY=/var/backups/borg/

if [[ -d /etc/apt ]]; then
    APT=true
    SUDO=sudo
    CRON=cron
elif [[ -d /etc/yum ]]; then
    YUM=true
    SUDO=wheel
    CRON=crond
else
	die 2 "Distribution not supported!"
fi


# Override defaults by sourcing config files. These are cumulative!
if [[ -f /etc/borgsetup ]]; then
	. /etc/borgsetup
fi
if [[ -f ~/.borgsetup ]]; then
	. ~/.borgsetup
fi

# parse command line options
parse-opt.flags SHOVE_BORG_REPO IMMEDIATE_BACKUP
parse-opt.params REMOTE_USER REMOTE_HOST REMOTE_DIRECTORY REMOTE_HOST_KEY REMOTE_BORG_PATH LOCAL_DIRECTORY SSH_KEYFILE PASSPHRASE_LENGTH
eval "$(parse-opt-simple)"

##########################
#### Let's go to work ####
##########################

if [[ ! "$(sudo bash -c 'echo $SSH_AUTH_SOCK')" ]]; then
	echo "sudo is not passing through \$SSH_AUTH_SOCK. This script will not work until you fix it."
	echo "Try setting 'Defaults env_keep += \"SSH_AUTH_SOCK\"' under /etc/sudoers.d"
	echo "Or did you forget to run monkeyproof?"
	exit 3
fi

HOSTNAME=$(hostname)
case "$HOSTNAME" in
*localhost*|*localdomain*|*dedibox* )
	die 4 "Your hostname $HOSTNAME is not configured properly, aborting"
	;;
esac

# If LOCAL_DIRECTORY is defined, it overrides remote configuration
if [[ "${LOCAL_DIRECTORY:-}" ]]; then
	if [[ "$LOCAL_DIRECTORY" = "/" ]]; then
		die 6 "Don't use / as your backup location! Aborting."
	fi
	REPO_LOCATION="$LOCAL_DIRECTORY"
elif [[ "${REMOTE_HOST:-}" ]]; then
	REPO_LOCATION="ssh://${REMOTE_USER}@${REMOTE_HOST}/./${REMOTE_DIRECTORY}"
else
	die 7 "Need to configure either a remote or local repo"
fi

# Canonicalise our location
case "$REPO_LOCATION" in
/*/ )
	# Local backup dir, ends in a /, nothing to do
	;;
/* )
	# Add a trailing /
	REPO_LOCATION=${REPO_LOCATION}/
	;;
*: )
	# Remote backup dir, ends in a :, nothing to do
	;;
*:*/ )
	# Remote backup dir, ends in a /, nothing to do
	;;
*:*)
	# Remote backup dir with a path not ending in a /
	REPO_LOCATION=${REPO_LOCATION}/
	;;
* )
	# Assume a remote location and add a trailing :
	REPO_LOCATION=${REPO_LOCATION}:
	;;
esac

export BORG_REPO="${REPO_LOCATION}${HOSTNAME}"

# very unfortunate naming convention mismatch
export BORG_REMOTE_PATH=$REMOTE_BORG_PATH

if [[ "${REMOTE_HOST:-}" ]]; then

    if [[ "${REMOTE_HOST_KEY:-}" ]]; then
        # Preseed our (and root's!) known_hosts with the remote host key
        echo "$REMOTE_HOST_KEY" >> ~/.ssh/known_hosts
        echo "$REMOTE_HOST_KEY" >> /root/.ssh/known_hosts
    fi

    # Make sure the remote host is usable and the parent directory exists
    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" -- mkdir -p "${REMOTE_DIRECTORY}"; then
        die 8 "Can't run necessary commands on remote host, aborting"
    fi

    # Sanity check for pre-existing repo
    if ssh "${REMOTE_USER}@${REMOTE_HOST}" -- test -d "${REMOTE_DIRECTORY}/${HOSTNAME:-NULL}"; then
        echo "Repo already exists"
        if [[ "${SHOVE_BORG_REPO:-}" == "true" ]]; then
            echo "Moving old repo aside"
            ssh "$REMOTE_USER@$REMOTE_HOST" -- mv "${REMOTE_DIRECTORY}/${HOSTNAME:-NULL}"{,."$(date +%s)"}
        else
            RECOVERY=true
        fi
    fi

else

    # Sanity check for pre-existing repo
    if test -d "${BORG_REPO}"; then
        echo "Repo already exists"
        if [[ "${SHOVE_BORG_REPO:-}" == "true" ]]; then
            echo "Moving old repo aside"
            mv "${BORG_REPO}"{,."$(date +%s)"}
        else
            RECOVERY=true
        fi
    else
        # Make sure parent directory exists
        mkdir -p "${REPO_LOCATION}"
    fi

fi

if [[ "${RECOVERY:-}" == "true" ]]; then
    echo "Entering recovery mode..."
    # In recovery mode, we must have a pre-existing configuration
    if [[ ! "${BORG_PASSPHRASE:-}" ]]; then
        export BORG_PASSPHRASE=$($YQ -r .storage.encryption_passphrase </etc/borgmatic/config.yaml)
        [[ "${BORG_PASSPHRASE:-}" ]] || die 5 "Recovery mode requires BORG_PASSPHRASE to be set; aborting"
    fi
    BORG_KEYFILE=/root/.config/borg/keys/$(ls -tr /root/.config/borg/keys | tail -1)
    if [[ ! "${BORG_KEYFILE:-}" ]]; then
        die 5 "Recovery mode requires a borg keyfile under /root/.config/borg/keys; aborting"
    fi
else
    export BORG_PASSPHRASE=$(< /dev/urandom tr -dc A-Za-z0-9 | head "-c$PASSPHRASE_LENGTH")
fi

if [[ "${APT:-}" == "true" ]]; then
	if [[ -z "$(find /var/cache/apt/pkgcache.bin -mmin -60)" ]]; then
		apt-get update
	fi
	apt-get -y install borgbackup python3-pip
    # try installing borgmatic from distro; fall back to pip
    apt-get -y install borgmatic \
        || pip3 install borgmatic \
        || die 11 "Could not install borgmatic; aborting."
elif [[ "${YUM:-}" == "true" ]]; then
	yum -y --setopt=skip_missing_names_on_install=False install borgbackup python34-pip
    # do not install borgmatic from distro; always use pip
    yum -y --setopt=skip_missing_names_on_install=False remove borgmatic || true
    pip3 install borgmatic \
        || die 11 "Could not install borgmatic; aborting."
fi

if ! $YQ . </dev/null 2>/dev/null; then
    pip3 install yq || die 12 "Could not install yq; aborting."
fi

if [[ ! -x "$BORGMATIC" ]]; then
    # Temporarily set PATH to be promiscuous so that we can find borgmatic
    PATH_SAVE=$PATH
    PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:$PATH
    BORGMATIC=$(which borgmatic 2>/dev/null || true)
    [[ -n "${BORGMATIC:-}" ]] || die 9 "Cannot find borgmatic anywhere; aborting."
    # We need to escape $PATH so that it is evaluated by cron, not here
    # beware the trailing space!
    ADDITIONAL_PATH="PATH=\$PATH:$(dirname "$BORGMATIC") "
    PATH=$PATH_SAVE
fi

if [[ ! -d /etc/borgmatic ]]; then
	mkdir /etc/borgmatic
fi

# Randomise backups between 1am and 6am inclusive
let "RANDHOUR = $RANDOM % 5 + 1"
let "RANDMIN = $RANDOM % 60"

if [[ -f /usr/local/bin/borg-silencer.pl || -f "$SCRIPT_DIR/borg-silencer.pl" ]]; then
    if [[ ! -f /usr/local/bin/borg-silencer.pl ]]; then
        cp "$SCRIPT_DIR/borg-silencer.pl" /usr/local/bin/
        chmod +x /usr/local/bin/borg-silencer.pl
    fi
    cat > /etc/cron.d/borgmatic <<EOF
10 0 * * * root test -x /usr/bin/mysqldump && test -d /run/mysqld && mysqldump -A -R -E --single-transaction --add-drop-table --triggers > /var/backups/mysql
20 0 * * * root test -x /usr/bin/pg_dumpall && test -d /run/postgresql && cd /tmp && sudo -u postgres pg_dumpall > /var/backups/postgres
$RANDMIN $RANDHOUR * * * root /bin/bash -c "$ADDITIONAL_PATH$BORGMATIC > >( /usr/local/bin/borg-silencer.pl 2>&1 ) 2>&1 | logger -t borgmatic"
EOF
else
    cat > /etc/cron.d/borgmatic <<EOF
10 0 * * * root test -x /usr/bin/mysqldump && test -d /run/mysqld && mysqldump -A -R -E --single-transaction --add-drop-table --triggers > /var/backups/mysql
20 0 * * * root test -x /usr/bin/pg_dumpall && test -d /run/postgresql && cd /tmp && sudo -u postgres pg_dumpall > /var/backups/postgres
$RANDMIN $RANDHOUR * * * root /bin/bash -c "$ADDITIONAL_PATH$BORGMATIC | logger -t borgmatic"
EOF
fi

service "$CRON" reload

if [[ -x /usr/bin/pg_dumpall && -d /run/postgresql ]]; then
	touch /var/backups/postgres && chown postgres /var/backups/postgres
fi

cat > /etc/borgmatic/excludes.src <<EOF
*.pyc
/home/*/.cache
*/lost+found
/var/cache
/var/lib/mysql
/var/lib/elasticsearch
*/nodes/*/indices
*/jfrog/artifactory/data
/var/lib/postgresql
*/jenkins/workspace
*/docker/tmp
*/docker/aufs
*/docker/overlay2
*/docker/containers
/tmp
/var/tmp
/usr/tmp
/proc
/run
/dev
/sys
*/u01
*/u02
EOF

curl -fsSL -o /etc/borgmatic/build-excludes https://raw.githubusercontent.com/andrewgdotcom/admin-tools/master/borgmatic-build-excludes
chmod +x /etc/borgmatic/build-excludes

cat > /etc/borgmatic/config.yaml <<EOF
location:
    # List of source directories to backup (required). Globs and tildes are expanded.
    source_directories:
EOF

# Force backup sources to exist; this works around a limitation in borgbackup
for dir in "${BACKUP_SOURCES[@]}"; do
    [[ -d "$dir" ]] || mkdir -p "$dir"
    cat <<EOF  >> /etc/borgmatic/config.yaml
        - $dir
EOF
done

cat >> /etc/borgmatic/config.yaml <<EOF
    # Stay in same file system (do not cross mount points).
    one_file_system: true

    # Alternate Borg remote executable. Defaults to "borg".
    remote_path: $REMOTE_BORG_PATH

    # Paths to local or remote repositories (required). Tildes are expanded. Multiple
    # repositories are backed up to in sequence. See ssh_command for SSH options like
    # identity file or port.
    repositories:
        - $BORG_REPO

    # Any paths matching these patterns are excluded from backups. Globs and tildes
    # are expanded. See the output of "borg help patterns" for more details.
    exclude_patterns:
        - '*.pyc'
        - /home/*/.cache
        - lost+found
        - /var/cache
        - /tmp
        - /var/tmp
        - /usr/tmp
        - /proc
        - /run
        - /dev
        - /sys

storage:
    # Passphrase to unlock the encryption key with. Only use on repositories that were
    # initialized with passphrase/repokey encryption. Quote the value if it contains
    # punctuation, so it parses correctly. And backslash any quote or backslash
    # literals as well.
    encryption_passphrase: $BORG_PASSPHRASE

    # Be tolerant of changes to the remote location.
    relocated_repo_access_is_ok: true

# At least one of the "keep" options is required for pruning to work.
retention:
    # Number of daily archives to keep.
    keep_daily: 7

    # Number of weekly archives to keep.
    keep_weekly: 4

    # Number of monthly archives to keep.
    keep_monthly: 6

    # Number of yearly archives to keep.
    keep_yearly: 1

consistency:
    # List of one or more consistency checks to run: "repository", "archives", and/or
    # "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
    # all consistency checks. "repository" checks the consistency of the repository,
    # "archive" checks all of the archives, and "extract" does an extraction dry-run
    # of just the most recent archive.
    checks:
        - repository
        - archives

    # Restrict the number of checked archives to the last n. Applies only to the "archives" check.
    check_last: 3
EOF

# If using local repos make some sanity checks
if [[ -z "${REMOTE_HOST:-}" ]]; then
	# Make sure the location exists
	if [[ ! -d "$REPO_LOCATION" ]]; then
		mkdir -p "$REPO_LOCATION"
	fi
	# Add our repo to the excludes, to prevent recursion
	echo "$BORG_REPO" >> /etc/borgmatic/excludes.src
fi

/etc/borgmatic/build-excludes || die 13 "Could not build excludes list"

# Make sure ordinary users can't read our passphrase etc.
chmod -R o= /etc/borgmatic

### Fire in the hole. ###

if [[ "${REMOTE_HOST:-}" ]]; then
	# Make ourselves a dedicated ssh key with no passphrase
	if [[ ! -f "$SSH_KEYFILE" ]]; then
		ssh-keygen -N "" -f "$SSH_KEYFILE" -C "borgmatic root@${HOSTNAME}"
	fi

	# Copy our ssh public key to the remote server. We need to be logged in to
	# this machine with agent forwarding enabled and an id already in
	# authorized_keys.
	#
	if ssh "${REMOTE_USER}@${REMOTE_HOST}" "/bin/sh -c exit"; then
		# Remote server appears to be fully capable
		ssh-copy-id -i "$SSH_KEYFILE" "${REMOTE_USER}@${REMOTE_HOST}"
	else
		# This is probably due to a restricted shell - try a stupider method
		# NB this is racy AF. Make sure you have a way to get back in (e.g. password)
		# TODO: try using sshfs?
		TEMPFILE=$(mktemp)
		scp "${REMOTE_USER}@${REMOTE_HOST}:.ssh/authorized_keys" "$TEMPFILE"
		cat "${SSH_KEYFILE}.pub" >> "$TEMPFILE"
		scp "$TEMPFILE" "${REMOTE_USER}@${REMOTE_HOST}:.ssh/authorized_keys"
		rm "$TEMPFILE"
	fi

	# Now make sure root uses the dedicated key for this connection
	if ! grep -q "Host\s\s*$REMOTE_HOST" /root/.ssh/config; then
		cat >> /root/.ssh/config <<EOF

Host $REMOTE_HOST
	User $REMOTE_USER
	IdentityFile $SSH_KEYFILE
EOF
	fi
fi

if [[ "${RECOVERY:-}" == "true" ]]; then
    echo "Testing the repository..."
    "$BORGMATIC" list > >( /usr/local/bin/borg-silencer.pl 2>&1 ) 2>&1
else
    # We need to su properly to root, otherwise borg makes a mess in our homedir
    # But don't use "su -", because we want to keep our other environment.
    su -c "borg init -e keyfile ${BORG_REPO}" || die 14 "Could not initialise repository"

    BORG_KEYFILE=/root/.config/borg/keys/$(ls -tr /root/.config/borg/keys | tail -1)

    if [[ "${IMMEDIATE_BACKUP:-}" == "true" ]]; then
        # use "su -" this time, because we want to emulate a cronny environment
        su - -c "$BORGMATIC" > /tmp/borg-immediate-backup.log 2>&1 &
    fi
fi

cat >> /usr/local/bin/borg-env <<EOF
export BORG_REPO=\$($YQ -r .location.repositories[0] </etc/borgmatic/config.yaml)
export BORG_PASSPHRASE=\$($YQ -r .storage.encryption_passphrase </etc/borgmatic/config.yaml)
export BORG_REMOTE_PATH=\$($YQ -r .remote_path </etc/borgmatic/config.yaml)
EOF

# Dump our info to stdout where e.g. ansible can pick it up
cat <<EOF

BORG AUTOCONFIGURATION SUCCEEDED

SAVE THE FOLLOWING INFO SOMEWHERE VERY VERY SAFE

Borg repo = $BORG_REPO
Encryption passphrase = $BORG_PASSPHRASE
Encryption keyfile = $BORG_KEYFILE , contents follow:
EOF
# force onto one line; ansible does fancy stuff with multiline output
cat "$BORG_KEYFILE" | tr '\r\n' ' '
cat <<EOF

The above has been saved in /root/.config/borg/keys and /root/.bashrc
EOF