From 3a0c14375f9270b7e8e8e5ad8455c951bca02a0a Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Fri, 2 Jul 2021 13:40:43 -0700 Subject: [PATCH 1/2] Update patroni documenation with api auth configuration Guide users to provide username/password for the api by default --- doc/administration/geo/setup/database.md | 18 ++++++++++++------ .../postgresql/replication_and_failover.md | 17 +++++++++++++++++ .../reference_architectures/10k_users.md | 6 +++++- .../reference_architectures/25k_users.md | 8 ++++++-- .../reference_architectures/3k_users.md | 8 ++++++-- .../reference_architectures/50k_users.md | 10 +++++++--- .../reference_architectures/5k_users.md | 6 +++++- 7 files changed, 58 insertions(+), 15 deletions(-) diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index a3b48476941847..03908e6fc45d4f 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -487,7 +487,7 @@ The replication process is now complete. PostgreSQL connections, which can improve performance even when using in a single instance installation. -We recommend using PgBouncer if you use GitLab in a highly available +We recommend using PgBouncer if you use GitLab in a highly available configuration with a cluster of nodes supporting a Geo **primary** site and two other clusters of nodes supporting a Geo **secondary** site. One for the main database and the other for the tracking database. For more information, @@ -550,12 +550,12 @@ Leader instance**: ```ruby roles(['patroni_role']) - + consul['services'] = %w(postgresql) consul['configuration'] = { retry_join: %w[CONSUL_PRIMARY1_IP CONSUL_PRIMARY2_IP CONSUL_PRIMARY3_IP] } - + # You need one entry for each secondary, with a unique name following PostgreSQL slot_name constraints: # # Configuration syntax is: 'unique_slotname' => { 'type' => 'physical' }, @@ -567,6 +567,8 @@ Leader instance**: patroni['use_pg_rewind'] = true patroni['postgresql']['max_wal_senders'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary). patroni['postgresql']['max_replication_slots'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary). + patroni['username'] = 'PATRONI_API_USERNAME' + patroni['password'] = 'PATRONI_API_PASSWORD' patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD' # We list all secondary instances as they can all become a Standby Leader @@ -727,16 +729,18 @@ For each Patroni instance on the secondary site: patroni['standby_cluster']['host'] = 'INTERNAL_LOAD_BALANCER_PRIMARY_IP' patroni['standby_cluster']['port'] = INTERNAL_LOAD_BALANCER_PRIMARY_PORT patroni['standby_cluster']['primary_slot_name'] = 'geo_secondary' # Or the unique replication slot name you setup before + patroni['username'] = 'PATRONI_API_USERNAME' + patroni['password'] = 'PATRONI_API_PASSWORD' patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD' patroni['use_pg_rewind'] = true patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica patroni['postgresql']['max_replication_slots'] = 5 # A minimum of three for one replica, plus two for each additional replica - + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH' postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' postgresql['listen_address'] = '0.0.0.0' # You can use a public or VPC address here instead - + gitlab_rails['dbpassword'] = 'POSTGRESQL_PASSWORD' gitlab_rails['enable'] = true gitlab_rails['auto_migrate'] = false @@ -754,7 +758,7 @@ For each Patroni instance on the secondary site: - If you are configuring a Patroni standby cluster on a site that previously had a working Patroni cluster: ```shell - gitlab-ctl stop patroni + gitlab-ctl stop patroni rm -rf /var/opt/gitlab/postgresql/data /opt/gitlab/embedded/bin/patronictl -c /var/opt/gitlab/patroni/patroni.yaml remove postgresql-ha gitlab-ctl reconfigure @@ -900,6 +904,8 @@ For each Patroni instance on the secondary site for the tracking database: ] # Patroni configuration + patroni['username'] = 'PATRONI_API_USERNAME' + patroni['password'] = 'PATRONI_API_PASSWORD' patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD' patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index b6d2e36851dfd3..fe64d03d309df6 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -157,6 +157,13 @@ We will need the following password information for the application's database u sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME ``` +#### Patroni information + +We will need the following password information for the Patroni API: + +- `PATRONI_API_USERNAME`. A username for basic auth to the API +- `PATRONI_API_PASSWORD`. A password for basic auth to the API + #### PgBouncer information When using default setup, minimum configuration requires: @@ -236,6 +243,11 @@ postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' +# Replace PATRONI_API_USERNAME with a username for Patroni Rest API calls (use the same username in all nodes) +patroni['username'] = 'PATRONI_API_USERNAME' +# Replace PATRONI_API_PASSWORD with a password for Patroni Rest API calls (use the same password in all nodes) +patroni['password'] = 'PATRONI_API_PASSWORD' + # Sets `max_replication_slots` to double the number of database nodes. # Patroni uses one extra slot per node when initiating the replication. patroni['postgresql']['max_replication_slots'] = X @@ -555,6 +567,8 @@ gitlab_rails['auto_migrate'] = false postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' +patroni['username'] = 'patroni_api' +patroni['password'] = 'example-password' patroni['postgresql']['max_replication_slots'] = 6 patroni['postgresql']['max_wal_senders'] = 7 @@ -642,6 +656,9 @@ postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' # Patroni uses one extra slot per node when initiating the replication. patroni['postgresql']['max_replication_slots'] = 6 +patroni['username'] = 'patroni_api' +patroni['password'] = 'example-password' + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index f94a500746c573..dd3dd6306b8257 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -598,6 +598,10 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + # Set up basic authentication for the Patroni API (use the same username/password in all nodes). + patroni['username'] = '' + patroni['password'] = '' + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) @@ -1681,7 +1685,7 @@ On each node: # balancer. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Gitaly + # Gitaly gitaly['enable'] = true # Make Gitaly accept connections on all network interfaces. You must use diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index aed54b236cef35..bacdb2874ff520 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -600,6 +600,10 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + # Set up basic authentication for the Patroni API (use the same username/password in all nodes). + patroni['username'] = '' + patroni['password'] = '' + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) @@ -863,7 +867,7 @@ a node and change its status from primary to replica (and vice versa). redis_exporter['flags'] = { 'redis.addr' => 'redis://10.6.0.51:6379', 'redis.password' => 'redis-password-goes-here', - } + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1699,7 +1703,7 @@ On each node: # balancer. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Gitaly + # Gitaly gitaly['enable'] = true # Make Gitaly accept connections on all network interfaces. You must use diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 71ca67075d33c0..7ed9b4b01918f3 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -848,7 +848,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except Patroni and Consul roles(['patroni_role']) - + # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -866,7 +866,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - + # Configure the Consul agent consul['services'] = %w(postgresql) ## Enable service discovery for Prometheus @@ -882,6 +882,10 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + # Set up basic authentication for the Patroni API (use the same username/password in all nodes). + patroni['username'] = '' + patroni['password'] = '' + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 51c803303292fc..a12e5ab4ae800f 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -608,6 +608,10 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + # Set up basic authentication for the Patroni API (use the same username/password in all nodes). + patroni['username'] = '' + patroni['password'] = '' + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) @@ -872,7 +876,7 @@ a node and change its status from primary to replica (and vice versa). 'redis.addr' => 'redis://10.6.0.51:6379', 'redis.password' => 'redis-password-goes-here', } - + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` @@ -1703,7 +1707,7 @@ On each node: # balancer. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Gitaly + # Gitaly gitaly['enable'] = true # Make Gitaly accept connections on all network interfaces. You must use @@ -1929,7 +1933,7 @@ To configure the Sidekiq nodes, on each one: ## Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - # Monitoring + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 3456e1193bdca4..72a93d2edbf3e3 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -846,7 +846,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Sets `max_replication_slots` to double the number of database nodes. # Patroni uses one extra slot per node when initiating the replication. patroni['postgresql']['max_replication_slots'] = 8 - + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. @@ -873,6 +873,10 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + # Set up basic authentication for the Patroni API (use the same username/password in all nodes). + patroni['username'] = '' + patroni['password'] = '' + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) -- GitLab From c1eab4a121f56d831a2ed4aa7ec70a73fd2104da Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Thu, 8 Jul 2021 14:26:59 +0000 Subject: [PATCH 2/2] Apply 2 suggestion(s) to 1 file(s) --- doc/administration/postgresql/replication_and_failover.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index fe64d03d309df6..440242ab2a9dfa 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -567,8 +567,8 @@ gitlab_rails['auto_migrate'] = false postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' -patroni['username'] = 'patroni_api' -patroni['password'] = 'example-password' +patroni['username'] = 'PATRONI_API_USERNAME' +patroni['password'] = 'PATRONI_API_PASSWORD' patroni['postgresql']['max_replication_slots'] = 6 patroni['postgresql']['max_wal_senders'] = 7 @@ -656,8 +656,8 @@ postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' # Patroni uses one extra slot per node when initiating the replication. patroni['postgresql']['max_replication_slots'] = 6 -patroni['username'] = 'patroni_api' -patroni['password'] = 'example-password' +patroni['username'] = 'PATRONI_API_USERNAME' +patroni['password'] = 'PATRONI_API_PASSWORD' # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the -- GitLab