EXPERIMENT: Auto-tune PostgreSQL parameters

Author: pablomh

src/roles/postgresql/defaults/main.yml

@@ -9,6 +9,8 @@ postgresql_data_dir: /var/lib/pgsql/data
 
 postgresql_admin_password: "CHANGEME"
 
-postgresql_max_connections: 500
-postgresql_shared_buffers: 512MB
-postgresql_effective_cache_size: 1GB
+postgresql_pulp_connections_count: "{{ pulp_worker_count + pulp_content_service_worker_count + pulp_api_service_worker_count }}"
+
+postgresql_max_connections: "{{ ( postgresql_pulp_connections_count | int + ( foreman_puma_workers | int * foreman_database_pool | int ) / 250 ) | round(0, 'ceil') | int * 250 }}"
+postgresql_shared_buffers: "{{ ( ansible_facts['memtotal_mb'] / 8192 ) | round(0, 'ceil') | int }}GB"
+postgresql_effective_cache_size: "{{ ( ansible_facts['memtotal_mb'] / 4096 ) | round(0, 'ceil') | int }}GB"

src/roles/pulp/defaults/main.yaml

@@ -5,10 +5,6 @@ pulp_api_image: "{{ pulp_container_image }}:{{ pulp_container_tag }}"
 pulp_content_image: "{{ pulp_container_image }}:{{ pulp_container_tag }}"
 pulp_worker_image: "{{ pulp_container_image }}:{{ pulp_container_tag }}"
 
-pulp_worker_count: "{{ [8, ansible_facts['processor_nproc']] | min }}"
-pulp_content_service_worker_count: "{{ (2 * ([8, ansible_facts['processor_nproc']] | min)) + 1 }}"
-pulp_api_service_worker_count: "{{ ([4, ansible_facts['processor_nproc']] | min) + 1 }}"
-
 pulp_volumes:
   - /var/lib/pulp:/var/lib/pulp
 

src/vars/defaults.yml

@@ -5,3 +5,20 @@ tuning_profile: development
 flavor: katello
 features: []
 enabled_features: "{{ (flavor_features + features) }}"
+
+pulp_worker_count: "{{ [8, ansible_facts['processor_nproc']] | min }}"
+pulp_content_service_worker_count: "{{ (2 * ([8, ansible_facts['processor_nproc']] | min)) + 1 }}"
+pulp_api_service_worker_count: "{{ ([4, ansible_facts['processor_nproc']] | min) + 1 }}"
+
+# Puma threads calculation:
+# Max calculation is based on the default for MRI https://puma.io/puma6/#thread-pool
+# Min calculation is based on testing that showed that having the same value as the max led to
+# lower memory usage
+foreman_puma_threads_max: 5
+# Puma workers calculation:
+# CPU calculation is based on https://github.com/puma/puma/blob/master/docs/deployment.md#mri
+# Memory calculation is based on https://docs.gitlab.com/ee/install/requirements.html#puma-settings
+foreman_puma_workers: "{{ [32, ansible_facts['processor_nproc'] * 1.5, (ansible_facts['memtotal_mb'] / 1024) - 1.5] | min | int }}"
+
+# Database pool calculation is based on https://community.theforeman.org/t/rails-connection-pool-size-optimizations/36675
+foreman_database_pool: "{{ foreman_puma_threads_max | int + 4 }}"