packaging: Add a tool to perform vacuum on dwh db

This tool can perform various vacuum operations without specifying the
ovirt_engine_history db credentials.

From the tool usage:

    -a        - analyze
    -A        - analyze only
    -f        - full vacuum
    -t        - table(s) to vacuum. Use multi -t table for multi tables
    -v        - verbose

Change-Id: I4e77e32928b427a6838e76d1d37c963f7d954815
Bug-Url: https://bugzilla.redhat.com/1409766
Signed-off-by: Shirly Radco <sradco@redhat.com>

.gitignore

@@ -8,6 +8,7 @@ build/python-check.sh
 ovirt-engine-dwh.spec
 packaging/conf/settings.properties
 packaging/etc/ovirt-engine-dwhd.conf.d/README
+packaging/bin/dwh-prolog.sh
 packaging/services/ovirt-engine-dwhd/config.py
 packaging/services/ovirt-engine-dwhd/ovirt-engine-dwhd.conf
 packaging/services/ovirt-engine-dwhd/ovirt-engine-dwhd.systemd

Makefile

@@ -100,6 +100,7 @@ GENERATED = \
 	build/python-check.sh \
 	ovirt-engine-dwh.spec \
 	packaging/etc/ovirt-engine-dwhd.conf.d/README \
+	packaging/bin/dwh-prolog.sh \
 	packaging/services/ovirt-engine-dwhd/config.py \
 	packaging/services/ovirt-engine-dwhd/ovirt-engine-dwhd.conf \
 	packaging/services/ovirt-engine-dwhd/ovirt-engine-dwhd.systemd \
@@ -205,6 +206,9 @@ install-layout: \
 
 	install -dm 755 "$(DESTDIR)$(PKG_STATE_DIR)/backups"
 
+	install -d -m 755 "$(DESTDIR)$(BIN_DIR)"
+	ln -sf "$(PKG_DATA_DIR)/bin/dwh-vacuum.sh" "$(DESTDIR)$(BIN_DIR)/dwh-vacuum"
+
 all-dev:
 	rm -f $(GENERATED)
 	$(MAKE) \

ovirt-engine-dwh.spec.in

@@ -230,7 +230,11 @@ fi
 %dir %attr(-, %{engine_user}, %{engine_group}) %{_localstatedir}/lib/ovirt-engine-dwh/
 %dir %attr(-, %{engine_user}, %{engine_group}) %{_localstatedir}/log/ovirt-engine-dwh/
 %dir %{_sysconfdir}/ovirt-engine-dwh
+%{_bindir}/dwh-vacuum
 %{_datadir}/ovirt-engine-dwh/
+%{_datadir}/ovirt-engine-dwh/bin/dwh-prolog.sh
+%{_datadir}/ovirt-engine-dwh/bin/dwh-vacuum.sh
+%{_datadir}/ovirt-engine-dwh/bin/generate-pgpass.sh
 %{_javadir}/ovirt-engine-dwh/
 %{_sysconfdir}/ovirt-engine-dwh/ovirt-engine-dwhd.conf.d/
 %{_localstatedir}/lib/ovirt-engine-dwh/backups/

packaging/bin/dwh-prolog.sh.in

@@ -0,0 +1,37 @@
+
+export DWH_DEFAULTS="${DWH_DEFAULTS:-@DWH_DEFAULTS@}"
+export DWH_VARS="${DWH_VARS:-@DWH_VARS@}"
+PACKAGE_NAME="@PACKAGE_NAME@"
+PACKAGE_VERSION="@PACKAGE_VERSION@"
+DISPLAY_VERSION="@DISPLAY_VERSION@"
+
+die() {
+	local m="$1"
+	echo "FATAL: ${m}" >&2
+	exit 1
+}
+
+load_config() {
+
+	[ -r "${DWH_DEFAULTS}" ] || die "Can't load defaults file \"${DWH_DEFAULTS}\"."
+
+	for f in \
+		"${DWH_DEFAULTS}" \
+		"${DWH_VARS}" \
+		$([ -d "${DWH_VARS}.d" ] && find "${DWH_VARS}.d" -name '*.conf' | sort) \
+		; do
+
+		[ -r "${f}" ] && . "${f}"
+	done
+
+	[ -n "${OVIRT_JBOSS_HOME}" ] && JBOSS_HOME="${OVIRT_JBOSS_HOME}"
+
+	JAVA_HOME="$(/usr/share/ovirt-engine/bin/java-home)" || die "Cannot set JAVA_HOME"
+	export JAVA_HOME
+
+	# clean the class path
+	# jboss module loader will not run otherwise.
+	export CLASSPATH=""
+}
+
+load_config

packaging/bin/dwh-vacuum.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+. "$(dirname "$(readlink -f "$0")")"/generate-pgpass.sh
+
+usage() {
+    cat << __EOF__
+Usage $0:
+
+    -a          - run analyze, update optimizer statistics
+    -A          - run analyze only, only update optimizer stats, no vacuum
+    -f          - do full vacuuming
+    -t          - vacuum specific table
+    -v          - verbose output
+
+    -h --help   - this help message
+__EOF__
+}
+
+MIXED_PARAMS_ERR="Can not mix -A and -f, use only one of them"
+
+while getopts ":aAft:v" opt; do
+    case $opt in
+        a) ANALYZE=1
+        ;;
+        A) ANALYZE=
+           ANALYZE_ONLY=1
+            [[ -n $FULL ]] && die "$MIXED_PARAMS_ERR"
+        ;;
+        f) FULL=1
+            [[ -n $ANALYZE_ONLY ]] && die "$MIXED_PARAMS_ERR"
+        ;;
+        t) TABLES="${TABLES} -t $OPTARG"
+        ;;
+        v) VERBOSE=1
+        ;;
+        \?) usage && exit
+        ;;
+        :) die "-$OPTARG requires an argument"
+        ;;
+    esac
+done
+
+# setups with 'trust' may have empty passwords
+[[ -n $DWH_DB_PASSWORD ]] && generatePgPass
+
+vacuumdb \
+${ANALYZE+-z} \
+${ANALYZE_ONLY+-Z} \
+${FULL+-f} \
+${VERBOSE+-v -e} \
+${TABLES+$TABLES} \
+-h $DWH_DB_HOST \
+-p $DWH_DB_PORT \
+-U $DWH_DB_USER \
+-d $DWH_DB_DATABASE \
+-w

packaging/bin/generate-pgpass.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+. "$(dirname "$(readlink -f "$0")")"/dwh-prolog.sh
+
+[[ -z $DWH_DB_HOST ]]     || \
+[[ -z $DWH_DB_PORT ]]     || \
+[[ -z $DWH_DB_USER ]]     || \
+[[ -z $DWH_DB_DATABASE ]] && die "Can't parse the connection details"
+
+MYTEMP="$(mktemp -d)"
+
+generatePgPass() {
+	local password="$(echo "${DWH_DB_PASSWORD}" | sed -e 's/\\/\\\\/g' -e 's/:/\\:/g')"
+	export PGPASSFILE="${MYTEMP}/.pgpass"
+	touch "${PGPASSFILE}" || die "Can't create ${PGPASSFILE}"
+	chmod 0600 "${PGPASSFILE}" || die "Can't chmod ${PGPASSFILE}"
+
+	cat > "${PGPASSFILE}" << __EOF__
+${DWH_DB_HOST}:${DWH_DB_PORT}:${DWH_DB_DATABASE}:${DWH_DB_USER}:${password}
+__EOF__
+}
+
+cleanup() {
+	[ -n "${MYTEMP}" ] && rm -fr "${MYTEMP}" ]
+}
+trap cleanup 0

packaging/setup/ovirt_engine_setup/dwh/constants.py

@@ -168,6 +168,11 @@ class FileLocations(object):
         'backups',
     )
 
+    OVIRT_ENGINE_DWH_BINDIR = os.path.join(
+        PKG_DATA_DIR,
+        'bin',
+    )
+
     OVIRT_ENGINE_DB_MD5_DIR = os.path.join(
         PKG_STATE_DIR,
         'dwh_dbmd5',
@@ -181,6 +186,11 @@ class FileLocations(object):
         'Default.properties',
     )
 
+    OVIRT_DWH_VACUUM_TOOL = os.path.join(
+        OVIRT_ENGINE_DWH_BINDIR,
+        'dwh-vacuum.sh',
+    )
+
 
 @util.export
 class Stages(object):
@@ -344,6 +354,12 @@ def RESTORE_BACKUP_LATE(self):
     def DISCONNECT_EXISTING_DWH(self):
         return 'OVESETUP_DWH_DB/disconnectExistingDwh'
 
+    @osetupattrs(
+        answerfile=True,
+    )
+    def DWH_VACUUM_FULL(self):
+        return 'OVESETUP_DB/dwhVacuumFull'
+
 
 @util.export
 @util.codegen

packaging/setup/plugins/ovirt-engine-setup/ovirt-engine-dwh/db/__init__.py

@@ -22,13 +22,15 @@
 from . import connection
 from . import engine_connection
 from . import schema
+from . import vacuum
 
 
 @util.export
 def createPlugins(context):
     connection.Plugin(context=context)
     engine_connection.Plugin(context=context)
     schema.Plugin(context=context)
+    vacuum.Plugin(context=context)
 
 
 # vim: expandtab tabstop=4 shiftwidth=4

packaging/setup/plugins/ovirt-engine-setup/ovirt-engine-dwh/db/vacuum.py

@@ -0,0 +1,130 @@
+#
+# ovirt-engine-setup -- ovirt engine setup
+# Copyright (C) 2013-2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""Vacuum plugin."""
+
+import datetime
+import gettext
+
+from otopi import plugin
+from otopi import util
+
+from ovirt_engine_setup.dwh import constants as odwhcons
+from ovirt_engine_setup.engine_common import constants as oengcommcons
+
+from ovirt_setup_lib import dialog
+
+
+def _(m):
+    return gettext.dgettext(message=m, domain='ovirt-engine-dwh')
+
+
+@util.export
+class Plugin(plugin.PluginBase):
+    """Vacuum plugin."""
+
+    def __init__(self, context):
+        super(Plugin, self).__init__(context=context)
+
+    @plugin.event(
+        stage=plugin.Stages.STAGE_INIT,
+    )
+    def _init(self):
+        self.environment.setdefault(
+            odwhcons.DBEnv.DWH_VACUUM_FULL,
+            None
+        )
+
+    @plugin.event(
+        stage=plugin.Stages.STAGE_CUSTOMIZATION,
+        condition=lambda self: (
+            self.environment[
+                odwhcons.CoreEnv.ENABLE
+            ] and not self.environment[
+                odwhcons.DBEnv.NEW_DATABASE
+            ]
+        ),
+        before=(
+            oengcommcons.Stages.DIALOG_TITLES_E_DATABASE,
+        ),
+        after=(
+            oengcommcons.Stages.DB_CONNECTION_STATUS,
+            oengcommcons.Stages.DIALOG_TITLES_S_DATABASE,
+        ),
+    )
+    def _customization(self):
+        if self.environment[
+            odwhcons.DBEnv.DWH_VACUUM_FULL
+        ] is not None:
+            return
+
+        self.environment[
+            odwhcons.DBEnv.DWH_VACUUM_FULL
+        ] = dialog.queryBoolean(
+            dialog=self.dialog,
+            name='DWH_VACUUM_FULL',
+            # TODO try to supply some estimation on the amount
+            # of space we will need to read/write/remove if possible.
+            # some projects like check_postgres may supply that report
+            # already. See https://github.com/bucardo/check_postgres
+            note=_(
+                'Perform full vacuum on the oVirt engine history'
+                '\ndatabase {db}@{host}?'
+                '\nThis operation may take a while'
+                ' depending on this setup health and the'
+                '\nconfiguration of the db vacuum process.'
+                '\nSee'
+                ' https://www.postgresql.org/docs/9.0/static/sql-vacuum.html'
+                '\n(@VALUES@) [@DEFAULT@]: '
+            ).format(
+                db=self.environment[
+                    odwhcons.DBEnv.DATABASE
+                ],
+                host=self.environment[
+                    odwhcons.DBEnv.HOST
+                ],
+            ),
+            prompt=True,
+            default=False
+        )
+
+    @plugin.event(
+        stage=plugin.Stages.STAGE_MISC,
+        condition=lambda self: self.environment[
+            odwhcons.DBEnv.DWH_VACUUM_FULL
+        ],
+        after=(
+            odwhcons.Stages.DB_SCHEMA,
+        ),
+    )
+    def _vacuum(self):
+        self.logger.info(
+            _("Running vacuum full on the ovirt_engine_history schema")
+        )
+        start = datetime.datetime.now()
+        args = [
+            odwhcons.FileLocations.OVIRT_DWH_VACUUM_TOOL,
+            '-f',
+            '-v'
+        ]
+        self.execute(args=args)
+        self.logger.info(
+            _("Running vacuum full elapsed {secs}").format(
+                secs=datetime.datetime.now() - start,
+            )
+        )