From 459d3812892cc3dc37a35a77c58d35fe2edd8504 Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Sat, 30 Nov 2024 22:08:33 +0300 Subject: [PATCH] FluentBit log collection setup support for old-style playbooks (#58) * added 'check_database_access_permission: true' to configuration samples * custom fact_path defined * revert adding fact_path as it becomes deprecated * better rolling restart for db nodes * error fixes * Print the parameters on database creation * error fixes * fluentbit role support * extra variables added to all.example --- examples/fluentbit/ddl.sql | 24 ++++ examples/fluentbit/dynamic-config.txt | 5 + examples/fluentbit/dynamic-config.yaml | 115 ++++++++++++++++++ examples/fluentbit/fluent-bit.conf | 66 ++++++++++ examples/fluentbit/fluent-bit.service | 15 +++ examples/fluentbit/parsers.conf | 5 + examples/fluentbit/plugins.conf | 2 + examples/full-dualhomed/cluster-config.yaml | 1 + files/config-1node.yaml | 3 +- files/config-3nodes-extra.yaml | 3 +- files/config-3nodes.yaml | 10 ++ files/config-9nodes.yaml | 5 + fluentbit-all.yaml | 4 + group_vars/all.example | 15 ++- roles/ydbd_fluentbit/tasks/main.yml | 114 +++++++++++++++++ .../templates/fluentbit-config.j2 | 75 ++++++++++++ .../templates/fluentbit-parsers.j2 | 5 + .../templates/fluentbit-plugins.j2 | 2 + .../templates/fluentbit-service.j2 | 15 +++ .../templates/list_all_db_services.j2 | 14 +++ roles/ydbd_newdb/tasks/main.yml | 2 +- roles/ydbd_rolling_dynamic/tasks/main.yml | 8 ++ .../tasks/restart_dynamic.yml | 59 ++++----- .../templates/list_db_services.j2 | 20 +++ run-active-options.sh | 4 +- run-fluentbit-all.sh | 4 + 26 files changed, 558 insertions(+), 37 deletions(-) create mode 100644 examples/fluentbit/ddl.sql create mode 100644 examples/fluentbit/dynamic-config.txt create mode 100644 examples/fluentbit/dynamic-config.yaml create mode 100644 examples/fluentbit/fluent-bit.conf create mode 100644 examples/fluentbit/fluent-bit.service create mode 100644 examples/fluentbit/parsers.conf create mode 100644 examples/fluentbit/plugins.conf create mode 100644 fluentbit-all.yaml create mode 100644 roles/ydbd_fluentbit/tasks/main.yml create mode 100644 roles/ydbd_fluentbit/templates/fluentbit-config.j2 create mode 100644 roles/ydbd_fluentbit/templates/fluentbit-parsers.j2 create mode 100644 roles/ydbd_fluentbit/templates/fluentbit-plugins.j2 create mode 100644 roles/ydbd_fluentbit/templates/fluentbit-service.j2 create mode 100644 roles/ydbd_fluentbit/templates/list_all_db_services.j2 create mode 100644 roles/ydbd_rolling_dynamic/templates/list_db_services.j2 create mode 100755 run-fluentbit-all.sh diff --git a/examples/fluentbit/ddl.sql b/examples/fluentbit/ddl.sql new file mode 100644 index 0000000..7b8ac8b --- /dev/null +++ b/examples/fluentbit/ddl.sql @@ -0,0 +1,24 @@ + +CREATE USER fluentbit PASSWORD "...."; +GRANT CONNECT ON `/cluster1/admin` TO fluentbit; + +CREATE TABLE `ydblogs` ( + `ts` Timestamp NOT NULL, + `datahash` Uint64 NOT NULL, + `dbname` Text NOT NULL, + `hostname` Text NOT NULL, + `input` Text NOT NULL, + `ts_orig` Text, + `level` Text, + `unit` Text, + `service` Text, + `msg` Text, + PRIMARY KEY ( + `ts`, `datahash`, `dbname`, `hostname`, `input` + ) +) PARTITION BY HASH(`ts`, `dbname`, `hostname`, `input`) +WITH ( + STORE = COLUMN +); + +GRANT USE ON `/cluster1/admin/ydblogs` TO fluentbit; diff --git a/examples/fluentbit/dynamic-config.txt b/examples/fluentbit/dynamic-config.txt new file mode 100644 index 0000000..94fe02b --- /dev/null +++ b/examples/fluentbit/dynamic-config.txt @@ -0,0 +1,5 @@ +ydb auth get-token -f >ydbd-token +./ydbd -s grpcs://bigpig1:2135 --ca-file ../tls/ca.crt -f ydbd-token admin console configs dump-yaml >step0.yaml +cp step0.yaml step1.yaml +vi step1.yaml +ydb admin config replace -f step1.yaml diff --git a/examples/fluentbit/dynamic-config.yaml b/examples/fluentbit/dynamic-config.yaml new file mode 100644 index 0000000..70d5bf5 --- /dev/null +++ b/examples/fluentbit/dynamic-config.yaml @@ -0,0 +1,115 @@ +metadata: + kind: MainConfig + cluster: "" + version: 2 +config: + yaml_config_enabled: true + table_profiles_config: + table_profiles: + - name: default + compaction_policy: default + execution_policy: default + partitioning_policy: default + storage_policy: default + replication_policy: default + caching_policy: default + compaction_policies: + - name: default + execution_policies: + - name: default + partitioning_policies: + - name: default + auto_split: true + auto_merge: true + size_to_split: 2147483648 + storage_policies: + - name: default + column_families: + - storage_config: + sys_log: + preferred_pool_kind: ssd + log: + preferred_pool_kind: ssd + data: + preferred_pool_kind: ssd + replication_policies: + - name: default + caching_policies: + - name: default +allowed_labels: + node_id: + type: string + host: + type: string + tenant: + type: string +selector_config: +- description: cookie=testdb-01 + selector: + tenant: /cluster1/testdb + config: + shared_cache_config: + memory_limit: 51539607552 + feature_flags: !inherit + enable_views: true + enable_data_shard_volatile_transactions: true + table_service_config: + sql_version: 1 + index_auto_choose_mode: MAX_USED_PREFIX + enable_implicit_query_parameter_types: true + enable_kqp_data_query_stream_lookup: true + enable_kqp_data_query_stream_idx_lookup_join: true + resource_manager: + kqp_pattern_cache_compiled_capacity_bytes: 524288000 + kqp_pattern_cache_capacity_bytes: 524288000 + query_limits: + result_rows_limit: 20000 + resource_broker_config: + queues: + - name: queue_restore + limit: + cpu: 8 + - name: queue_backup + limit: + cpu: 8 + actor_system_config: + executor: + - {name: System, spin_threshold: '10', threads: 2, max_threads: 11, type: BASIC} + - {name: User, spin_threshold: '1', threads: 11, max_threads: 15, type: BASIC} + - {name: Batch, spin_threshold: '1', threads: 2, max_threads: 6, type: BASIC} + - {name: IO, threads: 1, type: IO} + - {name: IC, spin_threshold: '10', threads: 7, max_threads: 11, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC} + scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'} + sys_executor: 0 + user_executor: 1 + batch_executor: 2 + io_executor: 3 + service_executor: + - {executor_id: 4, service_name: Interconnect} +- description: cookie=admin-01 + selector: + tenant: /cluster1/admin + config: + shared_cache_config: + memory_limit: 17179869184 + feature_flags: !inherit + enable_views: false + enable_data_shard_volatile_transactions: false + table_service_config: + sql_version: 1 + column_shard_config: + disabled_on_scheme_shard: false + actor_system_config: + executor: + - {name: System, spin_threshold: '10', threads: 1, max_threads: 4, type: BASIC} + - {name: User, spin_threshold: '1', threads: 3, max_threads: 7, type: BASIC} + - {name: Batch, spin_threshold: '1', threads: 1, max_threads: 4, type: BASIC} + - {name: IO, threads: 1, type: IO} + - {name: IC, spin_threshold: '10', threads: 2, max_threads: 4, time_per_mailbox_micro_secs: 100, max_avg_ping_deviation: 500, type: BASIC} + scheduler: {progress_threshold: '10000', resolution: '64', spin_threshold: '0'} + sys_executor: 0 + user_executor: 1 + batch_executor: 2 + io_executor: 3 + service_executor: + - {executor_id: 4, service_name: Interconnect} diff --git a/examples/fluentbit/fluent-bit.conf b/examples/fluentbit/fluent-bit.conf new file mode 100644 index 0000000..589550f --- /dev/null +++ b/examples/fluentbit/fluent-bit.conf @@ -0,0 +1,66 @@ +[SERVICE] + flush 2 + log_level info + parsers_file parsers.conf + plugins_file plugins.conf + storage.path /opt/ydb/fluentbit/data/storage + +[INPUT] + name systemd + Tag ydbd-storage + storage.type filesystem + DB /opt/ydb/fluentbit/data/ydbd-storage.db + Systemd_Filter _SYSTEMD_UNIT=ydbd-storage.service + +[INPUT] + name systemd + Tag ydbd-admin + storage.type filesystem + DB /opt/ydb/fluentbit/data/ydbd-admin.db + Systemd_Filter _SYSTEMD_UNIT=ydbd-admin-a.service + +[FILTER] + Name parser + Match ydbd-* + Key_Name MESSAGE + Parser ydb + Reserve_Data On + Preserve_Key On + +[FILTER] + Name modify + Match ydbd-* + Condition Key_does_not_exist P_LEVEL + Set P_LEVEL unknown + Hard_copy MESSAGE P_MESSAGE + +[FILTER] + Name modify + Match ydbd-storage + Add P_DB /cluster1 + +[FILTER] + Name modify + Match ydbd-admin + Add P_DB /cluster1/admin + +[FILTER] + Name record_modifier + Match ydbd-* + Allowlist_key P_DB + Allowlist_key P_SERVICE + Allowlist_key P_DTTM + Allowlist_key P_LEVEL + Allowlist_key P_MESSAGE + Allowlist_key _HOSTNAME + Allowlist_key _SYSTEMD_UNIT + +[OUTPUT] + Name ydb + LogLevel error + ConnectionURL grpcs://ydbd-1.front.private:2135/cluster1/admin + CredentialsStatic fluentbit:password + Certificates /opt/ydb/cfg/ca.crt + Match ydbd-* + TablePath ydblogs + Columns {".timestamp": "ts",".input":"input",".hash":"datahash","P_DB":"dbname","P_SERVICE":"service","P_DTTM":"ts_orig","P_LEVEL":"level","P_MESSAGE":"msg","_HOSTNAME":"hostname","_SYSTEMD_UNIT":"unit"} diff --git a/examples/fluentbit/fluent-bit.service b/examples/fluentbit/fluent-bit.service new file mode 100644 index 0000000..9918a73 --- /dev/null +++ b/examples/fluentbit/fluent-bit.service @@ -0,0 +1,15 @@ +[Unit] +Description=Fluent Bit +Documentation=https://docs.fluentbit.io/manual/ +Requires=network.target +After=network.target + +[Service] +Type=simple +EnvironmentFile=-/etc/sysconfig/fluent-bit +EnvironmentFile=-/etc/default/fluent-bit +ExecStart=/usr/local/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/examples/fluentbit/parsers.conf b/examples/fluentbit/parsers.conf new file mode 100644 index 0000000..37b6fb7 --- /dev/null +++ b/examples/fluentbit/parsers.conf @@ -0,0 +1,5 @@ +[PARSER] + Name ydb + Format regex + Regex ^(?[^ ]*) :(?[^ ]*) (?[^ ]*): (?.*).*$ + diff --git a/examples/fluentbit/plugins.conf b/examples/fluentbit/plugins.conf new file mode 100644 index 0000000..fc38c20 --- /dev/null +++ b/examples/fluentbit/plugins.conf @@ -0,0 +1,2 @@ +[PLUGINS] + out_ydb.so /opt/ydb/fluentbit/lib/out_ydb.so diff --git a/examples/full-dualhomed/cluster-config.yaml b/examples/full-dualhomed/cluster-config.yaml index 40e07ed..2fc7d98 100644 --- a/examples/full-dualhomed/cluster-config.yaml +++ b/examples/full-dualhomed/cluster-config.yaml @@ -236,6 +236,7 @@ table_service_config: kqp_pattern_cache_capacity_bytes: 524288000 feature_flags: + check_database_access_permission: true enable_views: true enable_data_shard_volatile_transactions: true enable_grpc_audit: false diff --git a/files/config-1node.yaml b/files/config-1node.yaml index da326fc..91e73aa 100644 --- a/files/config-1node.yaml +++ b/files/config-1node.yaml @@ -107,8 +107,7 @@ table_service_config: enable_kqp_data_query_stream_lookup: true feature_flags: - enable_grpc_audit: false - enable_get_node_labels: true + check_database_access_permission: true enable_views: true log_config: diff --git a/files/config-3nodes-extra.yaml b/files/config-3nodes-extra.yaml index a69c9a1..3e98683 100644 --- a/files/config-3nodes-extra.yaml +++ b/files/config-3nodes-extra.yaml @@ -184,8 +184,7 @@ bootstrap_config: shared_cache_config: {memory_limit: '4294967296'} feature_flags: - enable_grpc_audit: false - enable_get_node_labels: true + check_database_access_permission: true enable_views: true log_config: diff --git a/files/config-3nodes.yaml b/files/config-3nodes.yaml index c840dae..5bad3d7 100644 --- a/files/config-3nodes.yaml +++ b/files/config-3nodes.yaml @@ -132,6 +132,7 @@ blob_storage_config: # configuration of static blobstorage group. - node_id: ycydb-s3 pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_3 + channel_profile_config: profile: - channel: @@ -145,26 +146,35 @@ channel_profile_config: pdisk_category: 1 storage_pool_kind: ssd profile_id: 0 + interconnect_config: start_tcp: true encryption_mode: OPTIONAL path_to_certificate_file: "/opt/ydb/certs/node.crt" path_to_private_key_file: "/opt/ydb/certs/node.key" path_to_ca_file: "/opt/ydb/certs/ca.crt" + grpc_config: cert: "/opt/ydb/certs/node.crt" key: "/opt/ydb/certs/node.key" ca: "/opt/ydb/certs/ca.crt" services_enabled: - legacy + +feature_flags: + check_database_access_permission: true + table_service_config: sql_version: 1 + bootstrap_config: shared_cache_config: {memory_limit: '4294967296'} + audit_config: file_backend: format: JSON file_path: "ydb-audit.log" + # Actor system config is appended to the configuration file by Ansible task. # Provided here in the commented form for reference #actor_system_config: diff --git a/files/config-9nodes.yaml b/files/config-9nodes.yaml index cfc5a6e..b4b1221 100644 --- a/files/config-9nodes.yaml +++ b/files/config-9nodes.yaml @@ -5,6 +5,7 @@ host_configs: - path: /dev/disk/by-partlabel/ydb_disk_1 type: SSD host_config_id: 1 + hosts: - host: ycydb-s1 node_id: 1 @@ -69,6 +70,7 @@ hosts: body: 1 data_center: 'zone-c' rack: '3' + domains_config: domain: - name: Domain0 @@ -190,6 +192,9 @@ grpc_config: bootstrap_config: shared_cache_config: {memory_limit: '4294967296'} +feature_flags: + check_database_access_permission: true + table_service_config: sql_version: 1 diff --git a/fluentbit-all.yaml b/fluentbit-all.yaml new file mode 100644 index 0000000..608e445 --- /dev/null +++ b/fluentbit-all.yaml @@ -0,0 +1,4 @@ +- hosts: all + any_errors_fatal: true + roles: + - role: ydbd_fluentbit diff --git a/group_vars/all.example b/group_vars/all.example index 4267346..b4c4454 100644 --- a/group_vars/all.example +++ b/group_vars/all.example @@ -75,6 +75,19 @@ ydb_congestion_setting: htcp # Optional module to load the congestion control algorithm ydb_congestion_module: tcp_htcp +# Fluentbit software with the included YDB plugin +ydb_fluentbit_archive: "../fluent-bit-ydb.tar.xz" + +# Destination database for logs delivery +ydb_fluentbit_destination: "grpcs://ycydb-s1:2135/Domain0/admin" + +# Logs delivery destination database trusted CA +ydb_fluentbit_ca: "/opt/ydb/cfg/ca.crt" + +# Logs delivery destination database credentials. +# See examples/fluentbit for DDL and permissions. +ydb_fluentbit_credentials: "fluentbit:password" + # Should only be defined when YDB cluster extension is performed. -# Must not be defined during the initial cluster setup. +# Must NOT be defined during the initial cluster setup. # ydb_cluster_extension: true diff --git a/roles/ydbd_fluentbit/tasks/main.yml b/roles/ydbd_fluentbit/tasks/main.yml new file mode 100644 index 0000000..39ad204 --- /dev/null +++ b/roles/ydbd_fluentbit/tasks/main.yml @@ -0,0 +1,114 @@ +--- +# ydbd fluentibit instance deployment + +- name: Create the YDB fluentbit instance directory + file: + state: directory + path: "{{ ydb_dir }}/fluentbit" + group: ydb + owner: ydb + mode: '0755' + +- name: Install the fluentbit binary package + ansible.builtin.unarchive: + creates: "{{ ydb_dir }}/fluentbit/bin/fluent-bit" + dest: "{{ ydb_dir }}/fluentbit" + group: bin + owner: root + src: "{{ ydb_fluentbit_archive }}" + extra_opts: "{{ ydb_unpack_options }}" + +- name: Ensure permissions for the YDB fluentbit instance directory + file: + state: directory + path: "{{ ydb_dir }}/fluentbit" + group: bin + owner: root + mode: '0755' + +- name: Create the YDB fluentbit configuration directory + file: + state: directory + path: "{{ ydb_dir }}/fluentbit/etc" + group: bin + owner: root + mode: '0755' + +- name: Create the YDB fluentbit data directory + file: + state: directory + path: "{{ ydb_dir }}/fluentbit/data" + group: bin + owner: root + mode: '0700' + +- name: Create the YDB fluentbit buffer directory + file: + state: directory + path: "{{ ydb_dir }}/fluentbit/data/buffer" + group: bin + owner: root + mode: '0700' + +- name: Generate the YDB fluentbit parsers files + template: + src: fluentbit-parsers.j2 + dest: "{{ ydb_dir }}/fluentbit/etc/parsers.conf" + owner: root + group: root + mode: '0644' + +- name: Generate the YDB fluentbit plugins files + template: + src: fluentbit-plugins.j2 + dest: "{{ ydb_dir }}/fluentbit/etc/plugins.conf" + owner: root + group: root + mode: '0644' + +- name: Generate the YDB fluentbit service files + template: + src: fluentbit-service.j2 + dest: "/etc/systemd/system/ydbd-fluentbit.service" + owner: root + group: root + mode: '0644' + +- name: Refresh systemd services configuration + ansible.builtin.systemd: + daemon_reload: true + +- name: Update the database service listing script + template: + src: list_all_db_services.j2 + dest: "{{ ydb_dir }}/home/list_all_db_services.sh" + group: ydb + owner: ydb + mode: '0755' + +- name: "Collect the per-node database services" + ansible.builtin.command: "{{ ydb_dir }}/home/list_all_db_services.sh {{ ydb_dbname }}" + register: ydbd_svc_data + +- name: "Extract the per-node database services as list" + set_fact: + ydbd_svc: "{{ ydbd_svc_data.stdout_lines }}" + +- name: Generate the YDB fluentbit config files + template: + src: fluentbit-config.j2 + dest: "{{ ydb_dir }}/fluentbit/etc/fluent-bit.conf" + owner: root + group: root + mode: '0644' + +- name: Start the YDB fluentbit services + ansible.builtin.systemd: + state: started + name: ydbd-fluentbit + any_errors_fatal: true + +- name: Enable the YDB fluentbit services for automatic startup + ansible.builtin.systemd: + enabled: true + name: ydbd-fluentbit diff --git a/roles/ydbd_fluentbit/templates/fluentbit-config.j2 b/roles/ydbd_fluentbit/templates/fluentbit-config.j2 new file mode 100644 index 0000000..2389ba7 --- /dev/null +++ b/roles/ydbd_fluentbit/templates/fluentbit-config.j2 @@ -0,0 +1,75 @@ +#jinja2: trim_blocks:False +[SERVICE] + flush 2 + log_level info + parsers_file parsers.conf + plugins_file plugins.conf + storage.path {{ ydb_dir }}/fluentbit/data/buffer + +[INPUT] + name systemd + Tag ydbd-storage + storage.type filesystem + DB {{ ydb_dir }}/fluentbit/data/ydbd-storage.db + Systemd_Filter _SYSTEMD_UNIT=ydbd-storage.service + + +{%- for x1 in ydbd_svc %} +[INPUT] + name systemd + Tag {{ x1 }} + storage.type filesystem + DB {{ ydb_dir }}/fluentbit/data/{{ x1 }}.db + Systemd_Filter _SYSTEMD_UNIT={{ x1 }}.service + +{%- endfor %} + +[FILTER] + Name parser + Match ydbd-* + Key_Name MESSAGE + Parser ydb + Reserve_Data On + Preserve_Key On + +[FILTER] + Name modify + Match ydbd-* + Condition Key_does_not_exist P_LEVEL + Set P_LEVEL unknown + Hard_copy MESSAGE P_MESSAGE + +[FILTER] + Name modify + Match ydbd-storage + Add P_DB /{{ ydb_domain }} + + +{%- for x2 in ydbd_svc %} +[FILTER] + Name modify + Match {{ x2 }} + Add P_DB /{{ ydb_domain }}/{{ x2 | regex_replace('^ydbd-(.*)-*$', '\\1') }} + +{%- endfor %} + +[FILTER] + Name record_modifier + Match ydbd-* + Allowlist_key P_DB + Allowlist_key P_SERVICE + Allowlist_key P_DTTM + Allowlist_key P_LEVEL + Allowlist_key P_MESSAGE + Allowlist_key _HOSTNAME + Allowlist_key _SYSTEMD_UNIT + +[OUTPUT] + Name ydb + LogLevel error + ConnectionURL {{ ydb_fluentbit_destination }} + CredentialsStatic {{ ydb_fluentbit_credentials }} + Certificates {{ ydb_fluentbit_ca }} + Match ydbd-* + TablePath ydblogs + Columns {".timestamp": "ts",".input":"input",".hash":"datahash","P_DB":"dbname","P_SERVICE":"service","P_DTTM":"ts_orig","P_LEVEL":"level","P_MESSAGE":"msg","_HOSTNAME":"hostname","_SYSTEMD_UNIT":"unit"} diff --git a/roles/ydbd_fluentbit/templates/fluentbit-parsers.j2 b/roles/ydbd_fluentbit/templates/fluentbit-parsers.j2 new file mode 100644 index 0000000..37b6fb7 --- /dev/null +++ b/roles/ydbd_fluentbit/templates/fluentbit-parsers.j2 @@ -0,0 +1,5 @@ +[PARSER] + Name ydb + Format regex + Regex ^(?[^ ]*) :(?[^ ]*) (?[^ ]*): (?.*).*$ + diff --git a/roles/ydbd_fluentbit/templates/fluentbit-plugins.j2 b/roles/ydbd_fluentbit/templates/fluentbit-plugins.j2 new file mode 100644 index 0000000..56b453e --- /dev/null +++ b/roles/ydbd_fluentbit/templates/fluentbit-plugins.j2 @@ -0,0 +1,2 @@ +[PLUGINS] + out_ydb.so {{ ydb_dir }}/fluentbit/lib/out_ydb.so diff --git a/roles/ydbd_fluentbit/templates/fluentbit-service.j2 b/roles/ydbd_fluentbit/templates/fluentbit-service.j2 new file mode 100644 index 0000000..44e38a0 --- /dev/null +++ b/roles/ydbd_fluentbit/templates/fluentbit-service.j2 @@ -0,0 +1,15 @@ +#jinja2: trim_blocks:False +[Unit] +Description=Fluent Bit instance for YDB logs collection +Documentation=https://docs.fluentbit.io/manual/ +Requires=network.target +After=network.target + +[Service] +Type=simple +Environment=LD_LIBRARY_PATH={{ ydb_dir }}/fluentbit/lib +ExecStart={{ ydb_dir }}/fluentbit/bin/fluent-bit -c {{ ydb_dir }}/fluentbit/etc/fluent-bit.conf +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/roles/ydbd_fluentbit/templates/list_all_db_services.j2 b/roles/ydbd_fluentbit/templates/list_all_db_services.j2 new file mode 100644 index 0000000..44c85c6 --- /dev/null +++ b/roles/ydbd_fluentbit/templates/list_all_db_services.j2 @@ -0,0 +1,14 @@ +#! /bin/sh +# List the services for all YDB databases on the current host. + +set +e +set +u + +(cd /etc/systemd/system && ls ydbd-*-*.service) | while read sn; do + STATE=`systemctl show $sn | grep UnitFileState=enabled $tfile` + if [ ! -z "$STATE" ]; then + basename "$sn" .service + fi +done + +# End Of File diff --git a/roles/ydbd_newdb/tasks/main.yml b/roles/ydbd_newdb/tasks/main.yml index 56b4500..b7723b0 100644 --- a/roles/ydbd_newdb/tasks/main.yml +++ b/roles/ydbd_newdb/tasks/main.yml @@ -17,7 +17,7 @@ group: ydb owner: ydb mode: '0700' - - name: Run the database creation script + - name: "Create the database {{ ydb_dbname }} on {{ ydb_pool_kind }}:{{ ydb_default_groups }}" command: "{{ ydb_dir }}/home/create_database.sh {{ inventory_hostname }} {{ ydb_dbname }} {{ ydb_pool_kind }} {{ ydb_default_groups }}" become: true become_user: ydb diff --git a/roles/ydbd_rolling_dynamic/tasks/main.yml b/roles/ydbd_rolling_dynamic/tasks/main.yml index 3fe98a0..0b00bd8 100644 --- a/roles/ydbd_rolling_dynamic/tasks/main.yml +++ b/roles/ydbd_rolling_dynamic/tasks/main.yml @@ -1,6 +1,14 @@ --- # ydbd dynamic nodes rolling restart +- name: Update the database service listing script + template: + src: list_db_services.j2 + dest: "{{ ydb_dir }}/home/list_db_services.sh" + group: ydb + owner: ydb + mode: '0755' + - name: YDB database nodes rolling restart include_tasks: "restart_dynamic.yml" loop: "{{ groups['ydbd_dynamic']|flatten(levels=1) }}" diff --git a/roles/ydbd_rolling_dynamic/tasks/restart_dynamic.yml b/roles/ydbd_rolling_dynamic/tasks/restart_dynamic.yml index d88fcaa..5d5fd73 100644 --- a/roles/ydbd_rolling_dynamic/tasks/restart_dynamic.yml +++ b/roles/ydbd_rolling_dynamic/tasks/restart_dynamic.yml @@ -1,39 +1,40 @@ --- # YDB restart dynamic nodes on a single host -- name: Restart the dynamic nodes on {{ dynnode_name }} - ansible.builtin.systemd: - state: restarted - name: "ydbd-{{ ydb_dbname }}-{{ item.instance }}" - throttle: 1 +- name: "Identify the services for database {{ ydb_dbname }} on {{ dynnode_name }}" delegate_to: "{{ dynnode_name }}" delegate_facts: true - loop: "{{ ydb_dynnodes }}" - any_errors_fatal: true + ansible.builtin.command: "{{ ydb_dir }}/home/list_db_services.sh {{ ydb_dbname }}" + register: ydbd_db_svc -- name: Transfer the secrets to {{ dynnode_name }} - copy: - src: secret - dest: "{{ ydb_dir }}/certs/secret" - group: ydb - owner: ydb - mode: '0700' +- name: "Restart the database services on {{ dynnode_name }}" + when: ydbd_db_svc.stdout_lines|length > 0 delegate_to: "{{ dynnode_name }}" delegate_facts: true any_errors_fatal: true + block: + - name: "Invoke the service restart on {{ dynnode_name }}" + any_errors_fatal: true + loop: "{{ ydbd_db_svc.stdout_lines }}" + ansible.builtin.systemd: + state: restarted + name: "{{ item.split(' ')[0] }}" + - name: "Transfer the secrets to {{ dynnode_name }}" + copy: + src: secret + dest: "{{ ydb_dir }}/certs/secret" + group: ydb + owner: ydb + mode: '0700' + - name: "Wait for the dynamic nodes to come up on {{ dynnode_name }}" + become: true + become_user: ydb + loop: "{{ ydbd_db_svc.stdout_lines }}" + ansible.builtin.command: + cmd: "{{ ydb_dir }}/home/wait_dynamic.sh {{ inventory_hostname }} {{ item.split(' ')[1] }} {{ ydb_dbname }}" + - name: "Cleanup the transferred secrets on {{ dynnode_name }}" + file: state=absent path={{ ydb_dir }}/certs/secret + - name: Additional delay to settle the dynamic nodes + ansible.builtin.pause: seconds={{ dynnode_restart_sleep_seconds }} -- name: Wait for the dynamic nodes to come up - command: "{{ ydb_dir }}/home/wait_dynamic.sh {{ inventory_hostname }} {{ 2136 + item.offset }} {{ ydb_dbname }}" - become: true - become_user: ydb - delegate_to: "{{ dynnode_name }}" - delegate_facts: true - loop: "{{ ydb_dynnodes }}" - -- name: Cleanup the transferred secrets at {{ dynnode_name }} - file: state=absent path={{ ydb_dir }}/certs/secret - delegate_to: "{{ dynnode_name }}" - delegate_facts: true - -- name: Additional delay to settle the dynamic nodes - ansible.builtin.pause: seconds={{ dynnode_restart_sleep_seconds }} +# End Of File diff --git a/roles/ydbd_rolling_dynamic/templates/list_db_services.j2 b/roles/ydbd_rolling_dynamic/templates/list_db_services.j2 new file mode 100644 index 0000000..2e85b9e --- /dev/null +++ b/roles/ydbd_rolling_dynamic/templates/list_db_services.j2 @@ -0,0 +1,20 @@ +#! /bin/sh +# List the services for the particular YDB database on the current host. + +set +e +set +u + +DB_NAME="$1" + +(cd /etc/systemd/system && ls ydbd-${DB_NAME}-*.service) | while read sn; do + tfile=`mktemp` + systemctl show $sn >$tfile + STATE=`grep UnitFileState $tfile | grep enabled` + PORT=`grep ExecStart= $tfile | sed -n 's|.*grpcs-port \([0-9][0-9]*\).*|\1|p'` + rm -f $tfile + if [ ! -z "$STATE" ]; then + echo "$sn" ${PORT} + fi +done + +# End Of File diff --git a/run-active-options.sh b/run-active-options.sh index 4a30269..6dd4d63 100644 --- a/run-active-options.sh +++ b/run-active-options.sh @@ -2,7 +2,7 @@ # Useful extra options: # -k for password prompt -RUN_ANSIBLE="ansible-playbook -b -f 20 -i hosts" -RUN_ANSIBLE_PARALLEL="ansible-playbook -b -i hosts" +RUN_ANSIBLE="ansible-playbook -b -i hosts" +RUN_ANSIBLE_PARALLEL="ansible-playbook -b -f 40 -i hosts" # End Of File \ No newline at end of file diff --git a/run-fluentbit-all.sh b/run-fluentbit-all.sh new file mode 100755 index 0000000..91a011f --- /dev/null +++ b/run-fluentbit-all.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +. ./run-active-options.sh +${RUN_ANSIBLE_PARALLEL} fluentbit-all.yaml