From 45f386fd8aec7412a37d687f244925bf2eb5010d Mon Sep 17 00:00:00 2001 From: Patrick Erdelt Date: Tue, 7 Jan 2025 16:19:15 +0100 Subject: [PATCH] Bexhoma: Tests for pool component --- bexhoma/configurations.py | 40 ++- k8s/deploymenttemplate-PGBouncer.yml | 97 +++++-- k8s/deploymenttemplate-PGBouncerReplica.yml | 283 ++++++++++++++++++++ k8s/deploymenttemplate-PGBouncerSidecar.yml | 256 ++++++++++++++++++ test-pool.sh | 28 +- 5 files changed, 669 insertions(+), 35 deletions(-) create mode 100644 k8s/deploymenttemplate-PGBouncerReplica.yml create mode 100644 k8s/deploymenttemplate-PGBouncerSidecar.yml diff --git a/bexhoma/configurations.py b/bexhoma/configurations.py index 60efd711..b603e888 100644 --- a/bexhoma/configurations.py +++ b/bexhoma/configurations.py @@ -1070,6 +1070,7 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): storageConfiguration = configuration #name_pvc = self.generate_component_name(app=app, component='storage', experiment=self.storage_label, configuration=configuration) name_pvc = self.generate_component_name(app=app, component='storage', experiment=self.storage_label, configuration=storageConfiguration) + name_pool = self.generate_component_name(app=app, component='pool', experiment=experiment, configuration=configuration) self.logger.debug('configuration.start_sut(name={})'.format(name)) deployments = self.experiment.cluster.get_deployments(app=app, component=component, experiment=experiment, configuration=configuration) if len(deployments) > 0: @@ -1086,6 +1087,7 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): list_of_workers.append(worker_full_name) list_of_workers_as_string = ",".join(list_of_workers) env['BEXHOMA_WORKER_LIST'] = list_of_workers_as_string + env['BEXHOMA_SUT_NAME'] = name if self.num_worker > 0: worker_full_name = "{name_worker}-{worker_number}.{worker_service}".format(name_worker=name_worker, worker_number=0, worker_service=name_worker) env['BEXHOMA_WORKER_FIRST'] = worker_full_name @@ -1248,7 +1250,7 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): for i_env,e in env.items(): dep['spec']['template']['spec']['containers'][i_container]['env'].append({'name':i_env, 'value':str(e)}) #print(pvc) if dep['kind'] == 'Service': - if dep['metadata']['name'] != 'bexhoma-service': + if dep['metadata']['name'] == 'bexhoma-worker': #!= 'bexhoma-service': if self.num_worker == 0: del result[key] continue @@ -1272,6 +1274,22 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): if 'name' in ports and ports['name'] != 'port-dbms': del result[key]['spec']['ports'][i] continue + if dep['metadata']['name'] == 'bexhoma-pool': #!= 'bexhoma-service': + dep['metadata']['name'] = name_pool + dep['metadata']['labels']['app'] = app + dep['metadata']['labels']['component'] = 'pool' + dep['metadata']['labels']['configuration'] = configuration + dep['metadata']['labels']['experiment'] = experiment + dep['metadata']['labels']['dbms'] = self.docker + dep['metadata']['labels']['volume'] = self.volume + for label_key, label_value in self.additional_labels.items(): + dep['metadata']['labels'][label_key] = str(label_value) + #dep['spec']['selector'] = dep['metadata']['labels'].copy() + dep['spec']['selector']['configuration'] = configuration + dep['spec']['selector']['experiment'] = experiment + dep['spec']['selector']['dbms'] = self.docker + dep['spec']['selector']['volume'] = self.volume + continue dep['metadata']['labels']['app'] = app dep['metadata']['labels']['component'] = component dep['metadata']['labels']['configuration'] = configuration @@ -1294,14 +1312,20 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): del result[key]['spec']['ports'][i] #print(pvc) if dep['kind'] == 'Deployment': - yaml_deployment = result[key] - dep['metadata']['name'] = name + if dep['metadata']['name'] == 'bexhoma-pool': + dep['metadata']['name'] = name_pool + dep['metadata']['labels']['component'] = 'pool' + else: + yaml_deployment = result[key] # this will be marked 'loaded' iff pvc exists + dep['metadata']['name'] = name + dep['metadata']['labels']['component'] = component dep['metadata']['labels']['app'] = app - dep['metadata']['labels']['component'] = component dep['metadata']['labels']['configuration'] = configuration dep['metadata']['labels']['experiment'] = experiment dep['metadata']['labels']['dbms'] = self.docker dep['metadata']['labels']['volume'] = self.volume + dep['metadata']['labels']['sut'] = name + dep['metadata']['labels']['pool'] = name_pool for label_key, label_value in self.additional_labels.items(): dep['metadata']['labels'][label_key] = str(label_value) dep['metadata']['labels']['experimentRun'] = str(self.num_experiment_to_apply_done+1) @@ -1309,10 +1333,11 @@ def start_sut(self, app='', component='sut', experiment='', configuration=''): dep['spec']['template']['metadata']['labels'] = dep['metadata']['labels'].copy() deployment = dep['metadata']['name'] appname = dep['spec']['template']['metadata']['labels']['app'] - self.sut_containers_deployed = [] - self.worker_containers_deployed = [] + if dep['metadata']['name'] != 'bexhoma-pool': + self.sut_containers_deployed = [] for i_container, container in reversed(list(enumerate(dep['spec']['template']['spec']['containers']))): - self.sut_containers_deployed.append(container['name']) + if dep['metadata']['name'] != name_pool: + self.sut_containers_deployed.append(container['name']) self.logger.debug('configuration.create_manifest_deployment({})'.format(env)) if not 'env' in dep['spec']['template']['spec']['containers'][i_container]: dep['spec']['template']['spec']['containers'][i_container]['env'] = [] @@ -1522,6 +1547,7 @@ def stop_sut(self, app='', component='sut', experiment='', configuration=''): self.stop_loading() if component == 'sut': self.stop_sut(app=app, component='worker', experiment=experiment, configuration=configuration) + self.stop_sut(app=app, component='pool', experiment=experiment, configuration=configuration) def get_host_gpus(self): """ Returns information about the sut's host GPUs. diff --git a/k8s/deploymenttemplate-PGBouncer.yml b/k8s/deploymenttemplate-PGBouncer.yml index 7a7d8116..54023020 100644 --- a/k8s/deploymenttemplate-PGBouncer.yml +++ b/k8s/deploymenttemplate-PGBouncer.yml @@ -15,62 +15,71 @@ apiVersion: v1 kind: Service metadata: labels: {app: bexhoma, component: sut, configuration: default, experiment: default} - name: bexhoma-service + name: bexhoma-sut spec: + type: ClusterIP + clusterIP: None ports: - {port: 9091, protocol: TCP, name: port-dbms, targetPort: 5432} - - {port: 9300, protocol: TCP, name: port-monitoring, targetPort: 9300} - selector: {app: bexhoma, component: sut, configuration: default, experiment: default} + selector: {app: bexhoma, component: pool, configuration: default, experiment: default} --- apiVersion: apps/v1 kind: Deployment metadata: - labels: {app: bexhoma, component: sut, configuration: default, experiment: default} - name: bexhoma-deployment-postgres + name: bexhoma-pool + labels: {app: bexhoma, component: pool, configuration: default, experiment: default} spec: - replicas: 1 + replicas: 3 selector: - matchLabels: {app: bexhoma, component: sut, configuration: default, experiment: default} + matchLabels: {app: bexhoma, component: pool, configuration: default, experiment: default} template: metadata: - labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + labels: {app: bexhoma, component: pool, configuration: default, experiment: default} spec: - automountServiceAccountToken: false - imagePullSecrets: - - {name: dockerhub} - nodeSelector: - tolerations: - #- key: "nvidia.com/gpu" - # effect: "NoSchedule" - terminationGracePeriodSeconds: 180 containers: - name: pool image: edoburu/pgbouncer + # connect via 'psql -U postgres -p 5432 -h localhost pgbouncer' + # SHOW STATS; + # SHOW POOLS; + # SHOW CLIENTS; + # SHOW SERVERS; env: # https://github.com/edoburu/docker-pgbouncer/blob/master/entrypoint.sh - name: DB_USER value: postgres - name: DB_PASSWORD value: postgres - name: DB_HOST - value: localhost + valueFrom: + fieldRef: + fieldPath: metadata.labels['pool'] + #value: $(expr $BEXHOMA_SUT_NAME) - name: DB_PORT - value: "5433" + value: "5432" - name: AUTH_TYPE - value: scram-sha-256 # remove/comment this line if using postgres:13 and lower + value: md5 # scram-sha-256 # remove/comment this line if using postgres:13 and lower - name: POOL_MODE - value: session # mode = transaction not working for YCSB + value: transaction # session # mode = transaction not working for YCSB - name: ADMIN_USERS value: postgres - #- name: POOL_SIZE - # value: "256" - name: DEFAULT_POOL_SIZE value: "256" - name: MIN_POOL_SIZE value: "32" - name: MAX_CLIENT_CONN value: "256" - #- name: RESERVE_POOL_SIZE - # value: "256" + - name: LOG_CONNECTIONS + value: "0" + - name: LOG_DISCONNECTIONS + value: "0" + - name: LOG_POOLER_ERRORS + value: "0" + - name: LOG_STATS + value: "0" + - name: RESERVE_POOL_SIZE + value: "16" + - name: MAX_PREPARED_STATEMENTS + value: "256" #readinessProbe: # exec: # command: @@ -85,6 +94,40 @@ spec: limits: {cpu: 16000m, memory: 128Gi} requests: {cpu: 4000m, memory: 4Gi} #, ephemeral-storage: "1536Gi"} +--- +apiVersion: v1 +kind: Service +metadata: + labels: {app: bexhoma, component: pool, configuration: default, experiment: default} + name: bexhoma-pool +spec: + ports: + - {port: 5432, protocol: TCP, name: port-dbms, targetPort: 5432} + - {port: 9300, protocol: TCP, name: port-monitoring, targetPort: 9300} + selector: {app: bexhoma, component: sut, configuration: default, experiment: default} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-deployment-postgres +spec: + replicas: 1 + selector: + matchLabels: {app: bexhoma, component: sut, configuration: default, experiment: default} + template: + metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + spec: + automountServiceAccountToken: false + imagePullSecrets: + - {name: dockerhub} + nodeSelector: + tolerations: + #- key: "nvidia.com/gpu" + # effect: "NoSchedule" + terminationGracePeriodSeconds: 180 + containers: - name: dbms image: postgres:16.1 env: @@ -93,7 +136,7 @@ spec: - name: PGDATA value: /var/lib/postgresql/data/pgdata - name: PGPORT - value: "5433" # remove/comment this line for default ports + value: "5432" # remove/comment this line for default ports # pg_ctl: cannot be run as root lifecycle: #postStart: @@ -121,11 +164,11 @@ spec: - -U - postgres - -p - - "5433" + - "5432" initialDelaySeconds: 15 periodSeconds: 60 ports: - - {containerPort: 5433} + - {containerPort: 5432} securityContext: allowPrivilegeEscalation: false #runAsNonRoot: true diff --git a/k8s/deploymenttemplate-PGBouncerReplica.yml b/k8s/deploymenttemplate-PGBouncerReplica.yml new file mode 100644 index 00000000..03477a1b --- /dev/null +++ b/k8s/deploymenttemplate-PGBouncerReplica.yml @@ -0,0 +1,283 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-storage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: shared +--- +apiVersion: v1 +kind: Service +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-pool +spec: + type: ClusterIP + clusterIP: None + ports: + - {port: 9091, protocol: TCP, name: port-dbms, targetPort: 5432} + selector: {app: bexhoma, component: pool, configuration: default, experiment: default} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bexhoma-pool + labels: {app: bexhoma, component: pool, configuration: default, experiment: default} +spec: + replicas: 3 + selector: + matchLabels: {app: bexhoma, component: pool, configuration: default, experiment: default} + template: + metadata: + labels: {app: bexhoma, component: pool, configuration: default, experiment: default} + spec: + containers: + - name: pool + image: edoburu/pgbouncer + # connect via 'psql -U postgres -p 5432 -h localhost pgbouncer' + # SHOW STATS; + # SHOW POOLS; + # SHOW CLIENTS; + # SHOW SERVERS; + env: # https://github.com/edoburu/docker-pgbouncer/blob/master/entrypoint.sh + - name: DB_USER + value: postgres + - name: DB_PASSWORD + value: postgres + - name: DB_HOST + value: $(expr $BEXHOMA_SUT_NAME) + - name: DB_PORT + value: "5432" + - name: AUTH_TYPE + value: md5 # scram-sha-256 # remove/comment this line if using postgres:13 and lower + - name: POOL_MODE + value: transaction # session # mode = transaction not working for YCSB + - name: ADMIN_USERS + value: postgres + - name: DEFAULT_POOL_SIZE + value: "256" + - name: MIN_POOL_SIZE + value: "32" + - name: MAX_CLIENT_CONN + value: "256" + - name: LOG_CONNECTIONS + value: "0" + - name: LOG_DISCONNECTIONS + value: "0" + - name: LOG_POOLER_ERRORS + value: "0" + - name: LOG_STATS + value: "0" + - name: RESERVE_POOL_SIZE + value: "16" + - name: MAX_PREPARED_STATEMENTS + value: "256" + #readinessProbe: + # exec: + # command: + # - pg_isready + # - -U + # - postgres + # initialDelaySeconds: 15 + # periodSeconds: 60 + ports: + - {containerPort: 5432} + resources: + limits: {cpu: 16000m, memory: 128Gi} + requests: {cpu: 4000m, memory: 4Gi} + #, ephemeral-storage: "1536Gi"} +--- +apiVersion: v1 +kind: Service +metadata: + labels: {app: bexhoma, component: database, configuration: default, experiment: default} + name: bexhoma-service +spec: + ports: + - {port: 9091, protocol: TCP, name: port-dbms, targetPort: 5432} + - {port: 9300, protocol: TCP, name: port-monitoring, targetPort: 9300} + selector: {app: bexhoma, component: sut, configuration: default, experiment: default} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-deployment-postgres +spec: + replicas: 1 + selector: + matchLabels: {app: bexhoma, component: sut, configuration: default, experiment: default} + template: + metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + spec: + automountServiceAccountToken: false + imagePullSecrets: + - {name: dockerhub} + nodeSelector: + tolerations: + #- key: "nvidia.com/gpu" + # effect: "NoSchedule" + terminationGracePeriodSeconds: 180 + containers: + - name: dbms + image: postgres:16.1 + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: PGPORT + value: "5432" # remove/comment this line for default ports + # pg_ctl: cannot be run as root + lifecycle: + #postStart: + # exec: + # command: ["/bin/sh", "-c", "echo 'Hello from the postStart handler' > /usr/share/message && cat /usr/share/message"] + preStop: + exec: + #command: ["/bin/sh", "-c", "gosu postgres pg_ctl stop -D /var/lib/postgresql/data -m fast"] + #command: ["/bin/sh", "-c", "gosu postgres pg_ctl stop -m fast"] + #command: ["echo 'PRESTOP' ;", "/bin/sh", "-c", "gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "echo 'Hello from the preStop handler'; gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["gosu postgres", "pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c"] + #args: ["gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "gosu postgres 'pg_ctl stop -m smart -t 120'"] + #command: ["/bin/sh", "-c", "gosu postgres '/usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120'"] + #command: ["/bin/sh -c 'gosu postgres /usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120'"] + command: ["/bin/sh", "-c", "gosu postgres /usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "echo 'Hello from the preStop handler' > /usr/share/message && cat /usr/share/message"] + #command: ["sh", "-c", "trap 'gosu postgres pg_ctl stop -m smart' SIGTERM; gosu postgres postgres"] # trap: SIGTERM: bad trap + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + - -p + - "5432" + initialDelaySeconds: 15 + periodSeconds: 60 + ports: + - {containerPort: 5432} + securityContext: + allowPrivilegeEscalation: false + #runAsNonRoot: true + #runAsUser: 1000 + #runAsGroup: 1000 + #capabilities: + # drop: + # - ALL + #readOnlyRootFilesystem: true #could not create lock file "/var/run/postgresql/.s.PGSQL.5432.lock": Read-only file system + resources: + limits: {cpu: 16000m, memory: 128Gi} + requests: {cpu: 1000m, memory: 1Gi} + #, ephemeral-storage: "1536Gi"} + volumeMounts: + - {mountPath: /data, name: benchmark-data-volume} + - {mountPath: /dev/shm, name: dshm} + - {mountPath: /var/lib/postgresql/data, name: benchmark-storage-volume} + args: [ + "-c", "max_worker_processes=64", + "-c", "max_parallel_workers=64", + "-c", "max_parallel_workers_per_gather=64", + "-c", "max_parallel_maintenance_workers=64", # only for PostgreSQL > 10 (?) + "-c", "max_wal_size=32GB", + "-c", "shared_buffers=64GB", + #"-c", "shared_memory_size=32GB", # read-only + "-c", "max_connections=2048", + "-c", "autovacuum_max_workers=10", + "-c", "autovacuum_vacuum_cost_limit=3000", + "-c", "vacuum_cost_limit=1000", + "-c", "checkpoint_completion_target=0.9", + "-c", "cpu_tuple_cost=0.03", + "-c", "effective_cache_size=64GB", + "-c", "maintenance_work_mem=2GB", + #"-c", "max_connections=1700", + #"-c", "random_page_cost=1.1", + "-c", "wal_buffers=1GB", + "-c", "work_mem=32GB", + #"-c", "huge_pages=on", + "-c", "temp_buffers=4GB", + "-c", "autovacuum_work_mem=-1", + "-c", "max_stack_depth=7MB", + "-c", "max_files_per_process=4000", + "-c", "effective_io_concurrency=32", + "-c", "wal_level=minimal", + "-c", "max_wal_senders=0", + "-c", "synchronous_commit=off", + "-c", "checkpoint_timeout=1h", + "-c", "checkpoint_warning=0", + "-c", "autovacuum=off", + "-c", "max_locks_per_transaction=64", + "-c", "max_pred_locks_per_transaction=64", + "-c", "default_statistics_target=1000", + "-c", "random_page_cost=60" + ] + # , "-c", "listen_addresses='*'", "-c", "logging_collector=on" + # , "-c", "pg_stat_statements.save=off", "-c", "pg_stat_statements.track=all", "-c", "shared_preload_libraries='pg_stat_statements'" + - name: cadvisor + image: gcr.io/cadvisor/cadvisor:v0.47.0 + args: ["--port", "9300", "--storage_duration", "20m0s", "--docker_only", "true", "--disable_metrics", "disk,network,tcp,advtcp,udp,sched,process,hugetlb", "--application_metrics_count_limit", "30", "--housekeeping_interval", "5s"] + ports: + - containerPort: 9300 + #hostPort: 9300 + name: http + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + #runAsNonRoot: true + #runAsUser: 1000 + #runAsGroup: 1000 + #capabilities: + # drop: + # - ALL + readOnlyRootFilesystem: true + resources: + requests: {cpu: 150m, memory: 200Mi} + limits: {cpu: 16000m, memory: 128Gi} + volumeMounts: + - name: rootfs + mountPath: /rootfs + readOnly: true + - name: var-run + mountPath: /var/run + readOnly: true + - name: sys + mountPath: /sys + readOnly: true + - name: docker + mountPath: /var/lib/docker + readOnly: true + - name: disk + mountPath: /dev/disk + readOnly: true + volumes: + - name: benchmark-data-volume + persistentVolumeClaim: {claimName: bexhoma-data} + - name: benchmark-storage-volume + persistentVolumeClaim: {claimName: bexhoma-storage} + - name: rootfs + hostPath: + path: / + - name: var-run + hostPath: + path: /var/run + - name: sys + hostPath: + path: /sys + - name: docker + hostPath: + path: /var/lib/docker + - name: disk + hostPath: + path: /dev/disk + - name: dshm + emptyDir: + medium: Memory diff --git a/k8s/deploymenttemplate-PGBouncerSidecar.yml b/k8s/deploymenttemplate-PGBouncerSidecar.yml new file mode 100644 index 00000000..cd3668b9 --- /dev/null +++ b/k8s/deploymenttemplate-PGBouncerSidecar.yml @@ -0,0 +1,256 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-storage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: shared +--- +apiVersion: v1 +kind: Service +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-service +spec: + ports: + - {port: 9091, protocol: TCP, name: port-dbms, targetPort: 5432} + - {port: 9300, protocol: TCP, name: port-monitoring, targetPort: 9300} + selector: {app: bexhoma, component: sut, configuration: default, experiment: default} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + name: bexhoma-deployment-postgres +spec: + replicas: 1 + selector: + matchLabels: {app: bexhoma, component: sut, configuration: default, experiment: default} + template: + metadata: + labels: {app: bexhoma, component: sut, configuration: default, experiment: default} + spec: + automountServiceAccountToken: false + imagePullSecrets: + - {name: dockerhub} + nodeSelector: + tolerations: + #- key: "nvidia.com/gpu" + # effect: "NoSchedule" + terminationGracePeriodSeconds: 180 + containers: + - name: pool + image: edoburu/pgbouncer + # connect via 'psql -U postgres -p 5432 -h localhost pgbouncer' + # SHOW STATS; + # SHOW POOLS; + # SHOW CLIENTS; + # SHOW SERVERS; + env: # https://github.com/edoburu/docker-pgbouncer/blob/master/entrypoint.sh + - name: DB_USER + value: postgres + - name: DB_PASSWORD + value: postgres + - name: DB_HOST + value: localhost + - name: DB_PORT + value: "5433" + - name: AUTH_TYPE + value: scram-sha-256 # remove/comment this line if using postgres:13 and lower + - name: POOL_MODE + value: session # mode = transaction not working for YCSB + - name: ADMIN_USERS + value: postgres + - name: DEFAULT_POOL_SIZE + value: "256" + - name: MIN_POOL_SIZE + value: "32" + - name: MAX_CLIENT_CONN + value: "256" + - name: LOG_CONNECTIONS + value: "0" + - name: LOG_DISCONNECTIONS + value: "0" + - name: LOG_POOLER_ERRORS + value: "0" + - name: LOG_STATS + value: "0" + - name: RESERVE_POOL_SIZE + value: "16" + - name: MAX_PREPARED_STATEMENTS + value: "256" + #readinessProbe: + # exec: + # command: + # - pg_isready + # - -U + # - postgres + # initialDelaySeconds: 15 + # periodSeconds: 60 + ports: + - {containerPort: 5432} + resources: + limits: {cpu: 16000m, memory: 128Gi} + requests: {cpu: 4000m, memory: 4Gi} + #, ephemeral-storage: "1536Gi"} + - name: dbms + image: postgres:16.1 + env: + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: PGPORT + value: "5433" # remove/comment this line for default ports + # pg_ctl: cannot be run as root + lifecycle: + #postStart: + # exec: + # command: ["/bin/sh", "-c", "echo 'Hello from the postStart handler' > /usr/share/message && cat /usr/share/message"] + preStop: + exec: + #command: ["/bin/sh", "-c", "gosu postgres pg_ctl stop -D /var/lib/postgresql/data -m fast"] + #command: ["/bin/sh", "-c", "gosu postgres pg_ctl stop -m fast"] + #command: ["echo 'PRESTOP' ;", "/bin/sh", "-c", "gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "echo 'Hello from the preStop handler'; gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["gosu postgres", "pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c"] + #args: ["gosu postgres pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "gosu postgres 'pg_ctl stop -m smart -t 120'"] + #command: ["/bin/sh", "-c", "gosu postgres '/usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120'"] + #command: ["/bin/sh -c 'gosu postgres /usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120'"] + command: ["/bin/sh", "-c", "gosu postgres /usr/lib/postgresql/16/bin/pg_ctl stop -m smart -t 120"] + #command: ["/bin/sh", "-c", "echo 'Hello from the preStop handler' > /usr/share/message && cat /usr/share/message"] + #command: ["sh", "-c", "trap 'gosu postgres pg_ctl stop -m smart' SIGTERM; gosu postgres postgres"] # trap: SIGTERM: bad trap + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + - -p + - "5433" + initialDelaySeconds: 15 + periodSeconds: 60 + ports: + - {containerPort: 5433} + securityContext: + allowPrivilegeEscalation: false + #runAsNonRoot: true + #runAsUser: 1000 + #runAsGroup: 1000 + #capabilities: + # drop: + # - ALL + #readOnlyRootFilesystem: true #could not create lock file "/var/run/postgresql/.s.PGSQL.5432.lock": Read-only file system + resources: + limits: {cpu: 16000m, memory: 128Gi} + requests: {cpu: 1000m, memory: 1Gi} + #, ephemeral-storage: "1536Gi"} + volumeMounts: + - {mountPath: /data, name: benchmark-data-volume} + - {mountPath: /dev/shm, name: dshm} + - {mountPath: /var/lib/postgresql/data, name: benchmark-storage-volume} + args: [ + "-c", "max_worker_processes=64", + "-c", "max_parallel_workers=64", + "-c", "max_parallel_workers_per_gather=64", + "-c", "max_parallel_maintenance_workers=64", # only for PostgreSQL > 10 (?) + "-c", "max_wal_size=32GB", + "-c", "shared_buffers=64GB", + #"-c", "shared_memory_size=32GB", # read-only + "-c", "max_connections=2048", + "-c", "autovacuum_max_workers=10", + "-c", "autovacuum_vacuum_cost_limit=3000", + "-c", "vacuum_cost_limit=1000", + "-c", "checkpoint_completion_target=0.9", + "-c", "cpu_tuple_cost=0.03", + "-c", "effective_cache_size=64GB", + "-c", "maintenance_work_mem=2GB", + #"-c", "max_connections=1700", + #"-c", "random_page_cost=1.1", + "-c", "wal_buffers=1GB", + "-c", "work_mem=32GB", + #"-c", "huge_pages=on", + "-c", "temp_buffers=4GB", + "-c", "autovacuum_work_mem=-1", + "-c", "max_stack_depth=7MB", + "-c", "max_files_per_process=4000", + "-c", "effective_io_concurrency=32", + "-c", "wal_level=minimal", + "-c", "max_wal_senders=0", + "-c", "synchronous_commit=off", + "-c", "checkpoint_timeout=1h", + "-c", "checkpoint_warning=0", + "-c", "autovacuum=off", + "-c", "max_locks_per_transaction=64", + "-c", "max_pred_locks_per_transaction=64", + "-c", "default_statistics_target=1000", + "-c", "random_page_cost=60" + ] + # , "-c", "listen_addresses='*'", "-c", "logging_collector=on" + # , "-c", "pg_stat_statements.save=off", "-c", "pg_stat_statements.track=all", "-c", "shared_preload_libraries='pg_stat_statements'" + - name: cadvisor + image: gcr.io/cadvisor/cadvisor:v0.47.0 + args: ["--port", "9300", "--storage_duration", "20m0s", "--docker_only", "true", "--disable_metrics", "disk,network,tcp,advtcp,udp,sched,process,hugetlb", "--application_metrics_count_limit", "30", "--housekeeping_interval", "5s"] + ports: + - containerPort: 9300 + #hostPort: 9300 + name: http + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + #runAsNonRoot: true + #runAsUser: 1000 + #runAsGroup: 1000 + #capabilities: + # drop: + # - ALL + readOnlyRootFilesystem: true + resources: + requests: {cpu: 150m, memory: 200Mi} + limits: {cpu: 16000m, memory: 128Gi} + volumeMounts: + - name: rootfs + mountPath: /rootfs + readOnly: true + - name: var-run + mountPath: /var/run + readOnly: true + - name: sys + mountPath: /sys + readOnly: true + - name: docker + mountPath: /var/lib/docker + readOnly: true + - name: disk + mountPath: /dev/disk + readOnly: true + volumes: + - name: benchmark-data-volume + persistentVolumeClaim: {claimName: bexhoma-data} + - name: benchmark-storage-volume + persistentVolumeClaim: {claimName: bexhoma-storage} + - name: rootfs + hostPath: + path: / + - name: var-run + hostPath: + path: /var/run + - name: sys + hostPath: + path: /sys + - name: docker + hostPath: + path: /var/lib/docker + - name: disk + hostPath: + path: /dev/disk + - name: dshm + emptyDir: + medium: Memory diff --git a/test-pool.sh b/test-pool.sh index a7b42c53..df884995 100755 --- a/test-pool.sh +++ b/test-pool.sh @@ -17,7 +17,8 @@ BEXHOMA_NODE_SUT="cl-worker11" BEXHOMA_NODE_LOAD="cl-worker19" -BEXHOMA_NODE_BENCHMARK="cl-worker19" +BEXHOMA_NODE_BENCHMARK="cl-worker11" +#BEXHOMA_NODE_BENCHMARK="cl-worker19" LOG_DIR="./logs_tests" mkdir -p $LOG_DIR @@ -210,6 +211,31 @@ wait_process "ycsb" +### YCSB Loader Test for Scaling the Pooler with number of incoming and outgoing connections (TestCases.md) +nohup python ycsb.py -ms 1 -tr \ + -sf 16 \ + -sfo 16 \ + --workload c \ + -dbms PGBouncer PostgreSQL \ + -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \ + -tb 131072 \ + -nlp 8 \ + -nlt 64 \ + -nlf 1 \ + -nbp 1 \ + -nbt 64 \ + -nbf 1 \ + -ne 1 \ + -nc 1 \ + -nci 64 \ + -nco 64 \ + -rst shared -rss 50Gi \ + run $LOG_DIR/test_ycsb_testcase_pgbouncer_tmp_8.log & + +wait_process "ycsb" + + +