diff --git a/ansible/roles/xroad-ss/tasks/ubuntu.yml b/ansible/roles/xroad-ss/tasks/ubuntu.yml index bbf84f179a..98f4a4f0b1 100644 --- a/ansible/roles/xroad-ss/tasks/ubuntu.yml +++ b/ansible/roles/xroad-ss/tasks/ubuntu.yml @@ -28,6 +28,7 @@ with_items: - { question: "xroad-common/username", value: "{{ xroad_ui_user }}" } - { question: "xroad-common/database-host", value: "{{ database_host }}" } + - { question: "xroad-common/proxy-memory", value: "d" } - { question: "xroad-common/admin-subject", value: "/CN={{ inventory_hostname }}" } - { question: "xroad-common/admin-altsubject", value: "IP:{{ ansible_default_ipv4.address }},DNS:{{ inventory_hostname }}" } - { question: "xroad-common/service-subject", value: "/CN={{ inventory_hostname }}" } diff --git a/src/packages/src/xroad/common/base/usr/share/xroad/scripts/_setup_memory.sh b/src/packages/src/xroad/common/base/usr/share/xroad/scripts/_setup_memory.sh new file mode 100755 index 0000000000..9467d24876 --- /dev/null +++ b/src/packages/src/xroad/common/base/usr/share/xroad/scripts/_setup_memory.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +die () { + echo >&2 "$@" + exit 1 +} + +function to_megabytes() { + if [[ $1 =~ ^([0-9]+)m$ ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ $1 =~ ^([0-9]+)g$ ]]; then + echo "${BASH_REMATCH[1]} * 1024" | bc + else + echo "" + fi +} + +get_params_line() { + grep --color=never "^${1}=" "/etc/xroad/services/local.properties" +} + +apply_memory_config(){ + local -r params_file="/etc/xroad/services/local.properties" + local -r params="$(get_params_line "$1")" + + local -r xms="-Xms$2" + local -r xmx="-Xmx$3" + + if [ -z "$params" ]; then + echo "$1=$xms $xmx" >> "$params_file" + else + local pattern="-Xms[0-9]+[mg]" + if [[ "$params" =~ $pattern ]]; then + sed -i -E "/^$1=/s/$pattern/$xms/" "$params_file" + else + sed -i "s/^$1=.*/\0 $xms/" "$params_file" + fi + + local pattern="-Xmx[0-9]+[mg]" + if [[ "$params" =~ $pattern ]]; then + sed -i -E "/^$1=/s/$pattern/$xmx/" "$params_file" + else + sed -i "s/^$1=.*/\0 $xmx/" "$params_file" + fi + fi + + local -r updated_params="$(get_params_line "$1")" + + if [ "$params" == "$updated_params" ]; then + echo "No changes for config line: $updated_params" + else + echo "Updated config line: $updated_params" + fi + +} + +function to_gigabytes_str() { + local mb=$1 + local gb="" + if [[ $1 =~ ^([0-9]+)g$ ]]; then + gb="${BASH_REMATCH[1]}" + elif [[ $1 =~ ^([0-9]+)m$ ]]; then + mb="${BASH_REMATCH[1]}" + fi + + if [[ -z "$gb" && "$mb" -lt "1024" ]]; then + echo "${mb}m" + elif [[ -z "$gb" && "$mb" -ge "1024" ]]; then + gb=$(($mb / 1024)) + echo "${gb}g" + else + echo "${gb}g" + fi +} + +function find_xms() { + if [[ "$1" =~ -Xms([0-9]+[mg]) ]]; then + echo "${BASH_REMATCH[1]}" + else + echo "" + fi +} + +function find_xmx() { + if [[ "$1" =~ -Xmx([0-9]+[mg]) ]]; then + echo "${BASH_REMATCH[1]}" + else + echo "" + fi +} + +function apply_percentile() { + if [ "$1" -gt "1000" ]; then + die "First argument must be less than or equal to 1000" + fi + echo $2*$1/1000 | bc +} + +#Returns total memory in megabytes +function get_total_memory() { + local total_memory="" + + if [ -f /sys/fs/cgroup/memory.max ]; then + # cgroup v2 + memory_limit=$(cat /sys/fs/cgroup/memory.max) + elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then + # cgroup v1 + memory_limit=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) + fi + + if [[ -z "$memory_limit" || "$memory_limit" == "max" ]]; then + total_memory=$(free --mega | awk '/Mem:/ {print $2}') + else + total_memory=$(($memory_limit / 1024 / 1024)) + fi + + echo "$total_memory" +} + +#Returns used memory in megabytes +function get_used_memory() { + echo "$(free --mega | awk '/Mem:/ {print $3}')" +} + +#Calculates memory based on total memory and given percentiles +#Arguments: +#1. Total memory +#2. Minimum allowed memory in megabytes +#3. Maximum allowed memory in megabytes +#4. global array variable: memory_config. Percentiles list for getting memory based on total memory. +#For example: ("4000:125" "8000:500") means if total memory is less than 4000 then result is 1/8 of total memory, if less than 8000 then half of total memory. +#should be listed in ascending order based on total memory part. +memory_config=() +calculate_recommended_memory() { + local -r total_mem="$1" + local -r min_val=$(to_megabytes "$2") + local -r max_val=$(to_megabytes "$3") + + local result="$max_val" + + for pair in "${memory_config[@]}"; do + + IFS=':' read -ra config <<< "$pair" + local limit=$(to_megabytes "${config[0]}") + if [[ "$total_mem" -lt "$limit" ]]; then + result=$(apply_percentile "${config[1]}" "$total_mem") + break + fi + done + + if [[ "$result" -gt "$max_val" ]]; then + result="$max_val" + fi + + if [[ "$result" -lt "$min_val" ]]; then + result="$min_val" + fi + + if [[ "$result" -ge "2048" ]]; then + #round it up to nearest gb + result=$(echo "scale=0; (($result+512)/1024)" | bc) + echo "${result}g" + else + echo "${result}m" + fi +} diff --git a/src/packages/src/xroad/common/proxy/etc/xroad/services/proxy.conf b/src/packages/src/xroad/common/proxy/etc/xroad/services/proxy.conf index aa3d2c08fb..6af2d25c2d 100644 --- a/src/packages/src/xroad/common/proxy/etc/xroad/services/proxy.conf +++ b/src/packages/src/xroad/common/proxy/etc/xroad/services/proxy.conf @@ -19,7 +19,7 @@ done CP="/usr/share/xroad/jlib/proxy.jar" -XROAD_PROXY_PARAMS=" -Xms100m -Xmx512m -XX:MaxMetaspaceSize=140m \ +XROAD_PROXY_PARAMS=" $(/usr/share/xroad/scripts/proxy_memory_helper.sh 'get-default') -XX:MaxMetaspaceSize=140m \ -Djavax.net.ssl.sessionCacheSize=10000 \ -Dlogback.configurationFile=/etc/xroad/conf.d/proxy-logback.xml \ -Dxroad.proxy.clientHandlers=${CLIENT_HANDLERS#?} \ diff --git a/src/packages/src/xroad/common/proxy/usr/share/xroad/scripts/proxy_memory_helper.sh b/src/packages/src/xroad/common/proxy/usr/share/xroad/scripts/proxy_memory_helper.sh new file mode 100755 index 0000000000..bb6d262c10 --- /dev/null +++ b/src/packages/src/xroad/common/proxy/usr/share/xroad/scripts/proxy_memory_helper.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +source /usr/share/xroad/scripts/_setup_memory.sh + +usage() +{ +cat << EOF +usage: $0 + +Check and update memory settings for proxy service. + +OPTIONS: + action Action to execute, supported values: + status - display current state: total memory, used memory, current memory configuration for proxy + get-default - displays default memory configuration for proxy service + get-recommended - displays recommended memory configuration for proxy service based on total memory + apply-default - applies default memory configuration for proxy service + apply-recommended - applies recommended memory configuration for proxy service based on total memory + apply - applies custom memory config, requires 2 arguments: min and max memory, for example: $0 apply 512m 5g + +EOF +} + +default_xms="100m" +default_xmx="512m" + +verify_config(){ + local -r xms=$(to_megabytes "$1") + local -r xmx=$(to_megabytes "$2") + + if [ -z "$xms" ]; then + die "Invalid first argument. Must be in format, for example 128m" + fi + + if [ -z "$xmx" ]; then + die "Invalid second argument. Must be in format, for example 3g" + fi + + if [ "$xms" -ge "$xmx" ]; then + die "First argument should be smaller than second" + fi +} + +find_used_by_proxy(){ + echo "$(ps -u xroad -o %mem,args | awk '/ProxyMain/ {print $1}')%" +} + +get_current_xms(){ + local -r ps_args="$(ps -u xroad -o args | grep ProxyMain)" + local -r local_params="$(get_params_line 'XROAD_PROXY_PARAMS')" + + local xms=$(find_xms "$local_params") + + if [ -z "$xms" ]; then + xms=$(find_xms "$ps_args") + fi + + echo "$xms" +} + +get_current_xmx(){ + local -r ps_args="$(ps -u xroad -o args | grep ProxyMain)" + local -r local_params="$(get_params_line 'XROAD_PROXY_PARAMS')" + + local xmx=$(find_xmx "$local_params") + + if [ -z "$xmx" ]; then + xmx=$(find_xmx "$ps_args") + fi + + echo "$xmx" +} + +get_recommended_xms(){ + local -r total_mem=$(get_total_memory) + + memory_config=("4g:49" "8g:63" "16g:125") + echo $(calculate_recommended_memory "$total_mem" "100m" "2g") +} + +get_recommended_xmx(){ + local -r total_mem=$(get_total_memory) + + memory_config=("4g:125" "8g:250" "16g:500" "31g:52") + echo $(calculate_recommended_memory "$total_mem" "512m" "16g") +} + +display_status(){ + local -r total_memory=$(get_total_memory) + local -r used_memory=$(get_used_memory) + local -r used_by_proxy=$(find_used_by_proxy) + local -r current_xms=$(get_current_xms) + local -r current_xmx=$(get_current_xmx) + local -r recommended_xms=$(get_recommended_xms) + local -r recommended_xmx=$(get_recommended_xmx) + + local -r total_memory_str=$(to_gigabytes_str "$total_memory") + local -r used_memory_str=$(($used_memory * 100 / total_memory)) + +cat << EOF +Status: + Total memory: ${total_memory_str} + Used: ${used_memory_str}% + Used by proxy service: ${used_by_proxy} + + Current proxy service memory config: ${current_xms} - ${current_xmx} + + Default config: ${default_xms} - ${default_xmx} + (Apply default config with '$0 apply-default') + + Recommended config based on total memory: ${recommended_xms} - ${recommended_xmx} + (Apply recommended config with '$0 apply-recommended') + +EOF + +} + +if [[ -z "$1" || "$1" == "status" ]]; then + display_status +elif [ "$1" == "get-current" ]; then + echo "$(get_current_xms) $(get_current_xmx)" +elif [ "$1" == "get-default" ]; then + echo "-Xms$default_xms -Xmx$default_xmx" +elif [ "$1" == "get-recommended" ]; then + echo "-Xms$(get_recommended_xms) -Xmx$(get_recommended_xmx)" +elif [ "$1" == "apply-default" ]; then + apply_memory_config "XROAD_PROXY_PARAMS" "$default_xms" "$default_xmx" +elif [ "$1" == "apply-recommended" ]; then + apply_memory_config "XROAD_PROXY_PARAMS" "$(get_recommended_xms)" "$(get_recommended_xmx)" +elif [ "$1" == "apply" ]; then + verify_config "$2" "$3" + apply_memory_config "XROAD_PROXY_PARAMS" "$2" "$3" +elif [ "$1" == "help" ]; then + usage +else + die "Unknown action. Use 'help' to get available actions" +fi + + diff --git a/src/packages/src/xroad/redhat/SPECS/xroad-base.spec b/src/packages/src/xroad/redhat/SPECS/xroad-base.spec index db366e9b3a..467ba9536b 100644 --- a/src/packages/src/xroad/redhat/SPECS/xroad-base.spec +++ b/src/packages/src/xroad/redhat/SPECS/xroad-base.spec @@ -95,6 +95,7 @@ rm -rf %{buildroot} /usr/share/xroad/scripts/_backup_restore_common.sh /usr/share/xroad/scripts/serverconf_migrations/add_acl.xsl /usr/share/xroad/scripts/_setup_db.sh +/usr/share/xroad/scripts/_setup_memory.sh /usr/share/xroad/scripts/xroad-base.sh /usr/share/xroad/db/liquibase-core.jar /usr/share/xroad/db/liquibase-core-*.jar diff --git a/src/packages/src/xroad/redhat/SPECS/xroad-proxy.spec b/src/packages/src/xroad/redhat/SPECS/xroad-proxy.spec index 6c70dcb4b6..7099f53603 100644 --- a/src/packages/src/xroad/redhat/SPECS/xroad-proxy.spec +++ b/src/packages/src/xroad/redhat/SPECS/xroad-proxy.spec @@ -112,6 +112,7 @@ rm -rf %{buildroot} /usr/share/xroad/scripts/autobackup_xroad_proxy_configuration.sh /usr/share/xroad/scripts/get_security_server_id.sh /usr/share/xroad/scripts/read_db_properties.sh +/usr/share/xroad/scripts/proxy_memory_helper.sh %doc /usr/share/doc/%{name}/LICENSE.txt %doc /usr/share/doc/%{name}/3RD-PARTY-NOTICES.txt %doc /usr/share/doc/%{name}/CHANGELOG.md diff --git a/src/packages/src/xroad/ubuntu/generic/xroad-proxy.postinst b/src/packages/src/xroad/ubuntu/generic/xroad-proxy.postinst index 62973da135..796f1cfa5e 100644 --- a/src/packages/src/xroad/ubuntu/generic/xroad-proxy.postinst +++ b/src/packages/src/xroad/ubuntu/generic/xroad-proxy.postinst @@ -24,6 +24,14 @@ function migrate_conf_value { fi } +function handle_error() { + ERR=$(/tmp/memory.err; then + handle_error + continue + else + break + fi + fi + done + db_stop + invoke-rc.d --quiet rsyslog try-restart || true invoke-rc.d --quiet xroad-confclient try-restart || true invoke-rc.d --quiet xroad-signer try-restart || true diff --git a/src/packages/src/xroad/ubuntu/generic/xroad-proxy.templates b/src/packages/src/xroad/ubuntu/generic/xroad-proxy.templates index 7bc0868491..0fff23086f 100644 --- a/src/packages/src/xroad/ubuntu/generic/xroad-proxy.templates +++ b/src/packages/src/xroad/ubuntu/generic/xroad-proxy.templates @@ -4,6 +4,20 @@ Default: 127.0.0.1:5432 Description: Insert database server connection string This will be used by the Security Server to connect to the database host. +Template: xroad-common/proxy-memory +Type: string +Default: d +Description: Insert proxy server memory config string + This will be used to setup initial and maximum heap size for the proxy service. + . + Allowed values: + . + d - default config: ${DEFAULT_XM} + . + r - recommended config: ${RECOMMENDED_XM} + . + - custom heap size constrains, for example '128m 2g' + Template: xroad-common/admin-subject Type: string Description: Insert admin interface TLS certificate subject name @@ -50,6 +64,12 @@ Description: Error during certificate generation, please fix issues output was: ${ERR} +Template: xroad-common/proxy-memory-error +Type: error +Description: Error applying selected memory config, please fix issues + output was: + ${ERR} + Template: xroad-common/username Type: string Default: