From 3413e046a7c886ba9f35ff50c3182a73190c4c9c Mon Sep 17 00:00:00 2001 From: Nailia Iskhakova Date: Thu, 24 Dec 2020 16:49:23 +0300 Subject: [PATCH 1/7] Update RA docs to use Patroni Starting from 13.7 PG12 is the default version for new installs and upgrades Repmgr is deprecated Signed-off-by: Nailia Iskhakova --- .../reference_architectures/10k_users.md | 108 ++++----------- .../reference_architectures/25k_users.md | 108 ++++----------- .../reference_architectures/2k_users.md | 2 +- .../reference_architectures/3k_users.md | 130 ++++-------------- .../reference_architectures/50k_users.md | 108 ++++----------- .../reference_architectures/5k_users.md | 130 ++++-------------- 6 files changed, 126 insertions(+), 460 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 69e17c28e543cc..7389e9f1d1a427 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -455,14 +455,13 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -477,15 +476,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -517,19 +520,16 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL secondary nodes -1. On both the secondary nodes, add the same configuration specified above for the primary node - with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially - and there's no need to attempt to register them as a primary node: +1. On both the secondary nodes, add the same configuration as specified above for the primary node: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -537,22 +537,23 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Configure the Consul agent consul['services'] = %w(postgresql) - # Specify if a node should attempt to be primary on initialization. - repmgr['master_on_initialization'] = false - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace with your network addresses postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -601,84 +602,25 @@ SSH in to the **primary node**: 1. Exit the database prompt by typing `\q` and Enter. -1. Verify the cluster is initialized with one node: +1. Check the status of the leader and cluster: ```shell - gitlab-ctl repmgr cluster show + gitlab-ctl patroni members ``` The output should be similar to the following: ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as ``. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -SSH in to the **secondary node**: - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s + | Cluster | Member | Host | Role | State | TL | Lag in MB | Pending restart | + |---------------|-----------------------------------|-----------|--------|---------|-----|-----------|-----------------| + | postgresql-ha | | 10.6.0.21 | Leader | running | 175 | | * | + | postgresql-ha | | 10.6.0.22 | | running | 175 | 0 | * | + | postgresql-ha | | 10.6.0.23 | | running | 175 | 0 | * | ``` -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly and the secondary nodes appear in the cluster: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to the following: - -```plaintext -Role | Name | Upstream | Connection String -----------+---------|-----------|------------------------------------------------ -* master | MASTER | | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the +If the 'State' column for any node doesn't say "running", check the [Troubleshooting section](troubleshooting.md) before proceeding. -Also, check that the `repmgr-check-master` command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](troubleshooting.md) before proceeding. -
Back to setup components diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 03e340bdb88ff3..d21aa4e71e72b2 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -455,14 +455,13 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -477,15 +476,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -517,19 +520,16 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL secondary nodes -1. On both the secondary nodes, add the same configuration specified above for the primary node - with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially - and there's no need to attempt to register them as a primary node: +1. On both the secondary nodes, add the same configuration as specified above for the primary node: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -537,22 +537,23 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Configure the Consul agent consul['services'] = %w(postgresql) - # Specify if a node should attempt to be primary on initialization. - repmgr['master_on_initialization'] = false - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace with your network addresses postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -601,84 +602,25 @@ SSH in to the **primary node**: 1. Exit the database prompt by typing `\q` and Enter. -1. Verify the cluster is initialized with one node: +1. Check the status of the leader and cluster: ```shell - gitlab-ctl repmgr cluster show + gitlab-ctl patroni members ``` The output should be similar to the following: ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as ``. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -SSH in to the **secondary node**: - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s + | Cluster | Member | Host | Role | State | TL | Lag in MB | Pending restart | + |---------------|-----------------------------------|-----------|--------|---------|-----|-----------|-----------------| + | postgresql-ha | | 10.6.0.21 | Leader | running | 175 | | * | + | postgresql-ha | | 10.6.0.22 | | running | 175 | 0 | * | + | postgresql-ha | | 10.6.0.23 | | running | 175 | 0 | * | ``` -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly and the secondary nodes appear in the cluster: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to the following: - -```plaintext -Role | Name | Upstream | Connection String -----------+---------|-----------|------------------------------------------------ -* master | MASTER | | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the +If the 'State' column for any node doesn't say "running", check the [Troubleshooting section](troubleshooting.md) before proceeding. -Also, check that the `repmgr-check-master` command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](troubleshooting.md) before proceeding. -
Back to setup components diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 6158fabb8bd221..e664d55fa1bbef 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -271,7 +271,7 @@ further configuration steps. ```ruby # Disable all components except PostgreSQL roles ['postgres_role'] - repmgr['enable'] = false + patroni['enable'] = false consul['enable'] = false prometheus['enable'] = false alertmanager['enable'] = false diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 670b86913cb7aa..18930b2f927e63 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -705,14 +705,13 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -727,15 +726,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['enable'] = true @@ -757,28 +760,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. You can list the current PostgreSQL primary, secondary nodes status via: - - ```shell - sudo /opt/gitlab/bin/gitlab-ctl repmgr cluster show - ``` - -1. Verify the GitLab services are running: - - ```shell - sudo gitlab-ctl status - ``` - - The output should be similar to the following: - - ```plaintext - run: consul: (pid 30593) 77133s; run: log: (pid 29912) 77156s - run: logrotate: (pid 23449) 3341s; run: log: (pid 29794) 77175s - run: node-exporter: (pid 30613) 77133s; run: log: (pid 29824) 77170s - run: postgres-exporter: (pid 30620) 77132s; run: log: (pid 29894) 77163s - run: postgresql: (pid 30630) 77132s; run: log: (pid 29618) 77181s - run: repmgrd: (pid 30639) 77132s; run: log: (pid 29985) 77150s - ```
@@ -788,19 +769,16 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL secondary nodes -1. On both the secondary nodes, add the same configuration specified above for the primary node - with an additional setting that will inform `gitlab-ctl` that they are standby nodes initially - and there's no need to attempt to register them as a primary node: +1. On both the secondary nodes, add the same configuration as specified above for the primary node: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -808,9 +786,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Configure the Consul agent consul['services'] = %w(postgresql) - # Specify if a node should attempt to be primary on initialization. - repmgr['master_on_initialization'] = false - # START user configuration # Please set the real values as explained in Required Information section # @@ -818,15 +793,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['enable'] = true @@ -876,84 +855,25 @@ SSH in to the **primary node**: 1. Exit the database prompt by typing `\q` and Enter. -1. Verify the cluster is initialized with one node: +1. Check the status of the leader and cluster: ```shell - gitlab-ctl repmgr cluster show + gitlab-ctl patroni members ``` The output should be similar to the following: ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as ``. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -SSH in to the **secondary node**: - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup + | Cluster | Member | Host | Role | State | TL | Lag in MB | Pending restart | + |---------------|-----------------------------------|-----------|--------|---------|-----|-----------|-----------------| + | postgresql-ha | | 10.6.0.31 | Leader | running | 175 | | * | + | postgresql-ha | | 10.6.0.32 | | running | 175 | 0 | * | + | postgresql-ha | | 10.6.0.33 | | running | 175 | 0 | * | ``` - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` - -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly and the secondary nodes appear in the cluster: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to the following: - -```plaintext -Role | Name | Upstream | Connection String -----------+---------|-----------|------------------------------------------------ -* master | MASTER | | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the +If the 'State' column for any node doesn't say "running", check the [Troubleshooting section](troubleshooting.md) before proceeding. -Also, check that the `repmgr-check-master` command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](troubleshooting.md) before proceeding. -
Back to setup components diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 91b3b022365817..e34b92096319a3 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -455,14 +455,13 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -477,15 +476,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -517,19 +520,16 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL secondary nodes -1. On both the secondary nodes, add the same configuration specified above for the primary node - with an additional setting (`repmgr['master_on_initialization'] = false`) that will inform `gitlab-ctl` that they are standby nodes initially - and there's no need to attempt to register them as a primary node: +1. On both the secondary nodes, add the same configuration as specified above for the primary node: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -537,22 +537,23 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Configure the Consul agent consul['services'] = %w(postgresql) - # Specify if a node should attempt to be primary on initialization. - repmgr['master_on_initialization'] = false - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace with your network addresses postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -601,84 +602,25 @@ SSH in to the **primary node**: 1. Exit the database prompt by typing `\q` and Enter. -1. Verify the cluster is initialized with one node: +1. Check the status of the leader and cluster: ```shell - gitlab-ctl repmgr cluster show + gitlab-ctl patroni members ``` The output should be similar to the following: ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as ``. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -SSH in to the **secondary node**: - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup + | Cluster | Member | Host | Role | State | TL | Lag in MB | Pending restart | + |---------------|-----------------------------------|-----------|--------|---------|-----|-----------|-----------------| + | postgresql-ha | | 10.6.0.21 | Leader | running | 175 | | * | + | postgresql-ha | | 10.6.0.22 | | running | 175 | 0 | * | + | postgresql-ha | | 10.6.0.23 | | running | 175 | 0 | * | ``` - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` - -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly and the secondary nodes appear in the cluster: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to the following: - -```plaintext -Role | Name | Upstream | Connection String -----------+---------|-----------|------------------------------------------------ -* master | MASTER | | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the +If the 'State' column for any node doesn't say "running", check the [Troubleshooting section](troubleshooting.md) before proceeding. -Also, check that the `repmgr-check-master` command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](troubleshooting.md) before proceeding. -
Back to setup components diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 6d7511f67dbe1b..9a7dac15e96808 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -704,14 +704,13 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -726,15 +725,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behaviorv + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['enable'] = true @@ -756,28 +759,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. You can list the current PostgreSQL primary, secondary nodes status via: - - ```shell - sudo /opt/gitlab/bin/gitlab-ctl repmgr cluster show - ``` - -1. Verify the GitLab services are running: - - ```shell - sudo gitlab-ctl status - ``` - - The output should be similar to the following: - - ```plaintext - run: consul: (pid 30593) 77133s; run: log: (pid 29912) 77156s - run: logrotate: (pid 23449) 3341s; run: log: (pid 29794) 77175s - run: node-exporter: (pid 30613) 77133s; run: log: (pid 29824) 77170s - run: postgres-exporter: (pid 30620) 77132s; run: log: (pid 29894) 77163s - run: postgresql: (pid 30630) 77132s; run: log: (pid 29618) 77181s - run: repmgrd: (pid 30639) 77132s; run: log: (pid 29985) 77150s - ```
@@ -787,19 +768,16 @@ in the second step, do not supply the `EXTERNAL_URL` value. #### PostgreSQL secondary nodes -1. On both the secondary nodes, add the same configuration specified above for the primary node - with an additional setting that will inform `gitlab-ctl` that they are standby nodes initially - and there's no need to attempt to register them as a primary node: +1. On both the secondary nodes, add the same configuration as specified above for the primary node: ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul + # Disable all components except PostgreSQL, Patroni, and Consul roles ['postgres_role'] # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -807,9 +785,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Configure the Consul agent consul['services'] = %w(postgresql) - # Specify if a node should attempt to be primary on initialization. - repmgr['master_on_initialization'] = false - # START user configuration # Please set the real values as explained in Required Information section # @@ -817,15 +792,19 @@ in the second step, do not supply the `EXTERNAL_URL` value. postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' + + # Enable Patroni + patroni['enable'] = true # Set `max_wal_senders` to one more than the number of database nodes in the cluster. # This is used to prevent replication from using up all of the # available database connections. - postgresql['max_wal_senders'] = 4 - postgresql['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 4 + patroni['postgresql']['max_replication_slots'] = 4 + # Specify Patroni failover behavior + patroni['use_pg_rewind'] = true # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) ## Enable service discovery for Prometheus consul['enable'] = true @@ -874,84 +853,25 @@ SSH in to the **primary node**: 1. Exit the database prompt by typing `\q` and Enter. -1. Verify the cluster is initialized with one node: +1. Check the status of the leader and cluster: ```shell - gitlab-ctl repmgr cluster show + gitlab-ctl patroni members ``` The output should be similar to the following: ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as ``. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -SSH in to the **secondary node**: - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup + | Cluster | Member | Host | Role | State | TL | Lag in MB | Pending restart | + |---------------|-----------------------------------|-----------|--------|---------|-----|-----------|-----------------| + | postgresql-ha | | 10.6.0.31 | Leader | running | 175 | | * | + | postgresql-ha | | 10.6.0.32 | | running | 175 | 0 | * | + | postgresql-ha | | 10.6.0.33 | | running | 175 | 0 | * | ``` - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` - -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly and the secondary nodes appear in the cluster: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to the following: - -```plaintext -Role | Name | Upstream | Connection String -----------+---------|-----------|------------------------------------------------ -* master | MASTER | | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host= user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the +If the 'State' column for any node doesn't say "running", check the [Troubleshooting section](troubleshooting.md) before proceeding. -Also, check that the `repmgr-check-master` command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](troubleshooting.md) before proceeding. -
Back to setup components -- GitLab From d838c8e4bc896b1a6a3257efaa5c30ad98c343e4 Mon Sep 17 00:00:00 2001 From: Grant Young Date: Tue, 5 Jan 2021 17:09:54 +0000 Subject: [PATCH 2/7] Update Patroni config to latest for RA docs --- .../reference_architectures/10k_users.md | 98 ++++------------- .../reference_architectures/25k_users.md | 98 ++++------------- .../reference_architectures/3k_users.md | 100 ++++-------------- .../reference_architectures/50k_users.md | 98 ++++------------- .../reference_architectures/5k_users.md | 100 ++++-------------- 5 files changed, 95 insertions(+), 399 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 7389e9f1d1a427..00814a0b8f446e 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -422,9 +422,9 @@ install the necessary dependencies from step 1, and add the GitLab package repository from step 2. When installing GitLab in the second step, do not supply the `EXTERNAL_URL` value. -#### PostgreSQL primary node +#### PostgreSQL nodes -1. SSH in to the PostgreSQL primary node. +1. SSH in to one of the PostgreSQL nodes. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -460,22 +460,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = '' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = '' # Enable Patroni patroni['enable'] = true @@ -484,80 +468,29 @@ in the second step, do not supply the `EXTERNAL_URL` value. # available database connections. patroni['postgresql']['max_wal_senders'] = 4 patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } - # - # END user configuration - ``` - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - - - -#### PostgreSQL secondary nodes - -1. On both the secondary nodes, add the same configuration as specified above for the primary node: - - ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + patroni['postgresql']['max_connections'] = 500 # Disable automatic database migrations gitlab_rails['auto_migrate'] = false # Configure the Consul agent + consul['enable'] = true consul['services'] = %w(postgresql) + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + # START user configuration + # Please set the real values as explained in Required Information section + # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. - # This is used to prevent replication from using up all of the - # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace with your network addresses + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -567,8 +500,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } + # + # END user configuration ``` +NOTE: +Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods this has a small chance of leading to data loss. +Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d21aa4e71e72b2..653223dced3f6a 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -422,9 +422,9 @@ install the necessary dependencies from step 1, and add the GitLab package repository from step 2. When installing GitLab in the second step, do not supply the `EXTERNAL_URL` value. -#### PostgreSQL primary node +#### PostgreSQL nodes -1. SSH in to the PostgreSQL primary node. +1. SSH in to one of the PostgreSQL nodes. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -460,22 +460,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = '' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = '' # Enable Patroni patroni['enable'] = true @@ -484,80 +468,29 @@ in the second step, do not supply the `EXTERNAL_URL` value. # available database connections. patroni['postgresql']['max_wal_senders'] = 4 patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } - # - # END user configuration - ``` - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - - - -#### PostgreSQL secondary nodes - -1. On both the secondary nodes, add the same configuration as specified above for the primary node: - - ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + patroni['postgresql']['max_connections'] = 500 # Disable automatic database migrations gitlab_rails['auto_migrate'] = false # Configure the Consul agent + consul['enable'] = true consul['services'] = %w(postgresql) + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + # START user configuration + # Please set the real values as explained in Required Information section + # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. - # This is used to prevent replication from using up all of the - # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace with your network addresses + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -567,8 +500,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } + # + # END user configuration ``` +NOTE: +Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods this has a small chance of leading to data loss. +Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 18930b2f927e63..5d34e0bb259fd7 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -672,9 +672,9 @@ install the necessary dependencies from step 1, and add the GitLab package repository from step 2. When installing GitLab in the second step, do not supply the `EXTERNAL_URL` value. -#### PostgreSQL primary node +#### PostgreSQL nodes -1. SSH in to the PostgreSQL primary node. +1. SSH in to one of the PostgreSQL nodes. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -702,7 +702,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -710,22 +710,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = '' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = '' # Enable Patroni patroni['enable'] = true @@ -734,57 +718,17 @@ in the second step, do not supply the `EXTERNAL_URL` value. # available database connections. patroni['postgresql']['max_wal_senders'] = 4 patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - postgres_exporter['dbname'] = 'gitlabhq_production' - postgres_exporter['password'] = '' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } - # - # END user configuration - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - - - -#### PostgreSQL secondary nodes - -1. On both the secondary nodes, add the same configuration as specified above for the primary node: - - ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + patroni['postgresql']['max_connections'] = 500 # Disable automatic database migrations gitlab_rails['auto_migrate'] = false # Configure the Consul agent + consul['enable'] = true consul['services'] = %w(postgresql) + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section @@ -794,37 +738,31 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. - # This is used to prevent replication from using up all of the - # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' - postgres_exporter['dbname'] = 'gitlabhq_production' - postgres_exporter['password'] = '' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } + # # END user configuration ``` +NOTE: +Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods this has a small chance of leading to data loss. +Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index e34b92096319a3..3600db200f6aba 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -422,9 +422,9 @@ install the necessary dependencies from step 1, and add the GitLab package repository from step 2. When installing GitLab in the second step, do not supply the `EXTERNAL_URL` value. -#### PostgreSQL primary node +#### PostgreSQL nodes -1. SSH in to the PostgreSQL primary node. +1. SSH in to one of the PostgreSQL nodes. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -460,22 +460,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = '' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = '' # Enable Patroni patroni['enable'] = true @@ -484,80 +468,29 @@ in the second step, do not supply the `EXTERNAL_URL` value. # available database connections. patroni['postgresql']['max_wal_senders'] = 4 patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } - # - # END user configuration - ``` - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - - - -#### PostgreSQL secondary nodes - -1. On both the secondary nodes, add the same configuration as specified above for the primary node: - - ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + patroni['postgresql']['max_connections'] = 500 # Disable automatic database migrations gitlab_rails['auto_migrate'] = false # Configure the Consul agent + consul['enable'] = true consul['services'] = %w(postgresql) + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + # START user configuration + # Please set the real values as explained in Required Information section + # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' - - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. - # This is used to prevent replication from using up all of the - # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - # Replace with your network addresses + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) - ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -567,8 +500,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } + # + # END user configuration ``` +NOTE: +Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods this has a small chance of leading to data loss. +Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. + 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, add the file from your Consul server to this server. diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 9a7dac15e96808..775d78eb903562 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -671,9 +671,9 @@ install the necessary dependencies from step 1, and add the GitLab package repository from step 2. When installing GitLab in the second step, do not supply the `EXTERNAL_URL` value. -#### PostgreSQL primary node +#### PostgreSQL nodes -1. SSH in to the PostgreSQL primary node. +1. SSH in to one of the PostgreSQL nodes. 1. Generate a password hash for the PostgreSQL username/password pair. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -701,7 +701,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the primary database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -709,22 +709,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = '' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = '' # Enable Patroni patroni['enable'] = true @@ -733,57 +717,17 @@ in the second step, do not supply the `EXTERNAL_URL` value. # available database connections. patroni['postgresql']['max_wal_senders'] = 4 patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behaviorv - patroni['use_pg_rewind'] = true - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on for monitoring - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - postgres_exporter['dbname'] = 'gitlabhq_production' - postgres_exporter['password'] = '' - - ## The IPs of the Consul server nodes - ## You can also use FQDNs and intermix them with IPs - consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), - } - # - # END user configuration - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - - - -#### PostgreSQL secondary nodes - -1. On both the secondary nodes, add the same configuration as specified above for the primary node: - - ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + patroni['postgresql']['max_connections'] = 500 # Disable automatic database migrations gitlab_rails['auto_migrate'] = false # Configure the Consul agent + consul['enable'] = true consul['services'] = %w(postgresql) + ## Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section @@ -793,37 +737,31 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. - # This is used to prevent replication from using up all of the - # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 - # Specify Patroni failover behavior - patroni['use_pg_rewind'] = true - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.6.0.0/24) - - ## Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true + postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24) # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' - postgres_exporter['dbname'] = 'gitlabhq_production' - postgres_exporter['password'] = '' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } + # # END user configuration ``` +NOTE: +Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods this has a small chance of leading to data loss. +Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace + the file of the same name on this server. If that file is not on this server, + add the file from your Consul server to this server. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) -- GitLab From d19c25642b9e4fdbb423ceb434ea512726868a96 Mon Sep 17 00:00:00 2001 From: Grant Young Date: Tue, 5 Jan 2021 17:21:15 +0000 Subject: [PATCH 3/7] Docs lint fixes for new RA PG config --- doc/administration/reference_architectures/10k_users.md | 6 +++--- doc/administration/reference_architectures/25k_users.md | 6 +++--- doc/administration/reference_architectures/3k_users.md | 6 +++--- doc/administration/reference_architectures/50k_users.md | 6 +++--- doc/administration/reference_architectures/5k_users.md | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 00814a0b8f446e..930dbf4d960555 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -505,9 +505,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` NOTE: -Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. +Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, @@ -578,7 +578,7 @@ The following IPs will be used as an example: 1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace `` and `` with the - password hashes you [set up previously](#postgresql-primary-node): + password hashes you [set up previously](#postgresql-nodes): ```ruby # Disable all components except Pgbouncer and Consul agent diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 653223dced3f6a..5c7003facd5383 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -505,9 +505,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` NOTE: -Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. +Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, @@ -578,7 +578,7 @@ The following IPs will be used as an example: 1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace `` and `` with the - password hashes you [set up previously](#postgresql-primary-node): + password hashes you [set up previously](#postgresql-nodes): ```ruby # Disable all components except Pgbouncer and Consul agent diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 5d34e0bb259fd7..6a04b42771450d 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -755,9 +755,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` NOTE: -Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. +Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, @@ -829,7 +829,7 @@ The following IPs will be used as an example: 1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace `` and `` with the - password hashes you [set up previously](#postgresql-primary-node): + password hashes you [set up previously](#postgresql-nodes): ```ruby # Disable all components except Pgbouncer and Consul agent diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 3600db200f6aba..c2f75db417b150 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -505,9 +505,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` NOTE: -Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. +Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, @@ -578,7 +578,7 @@ The following IPs will be used as an example: 1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace `` and `` with the - password hashes you [set up previously](#postgresql-primary-node): + password hashes you [set up previously](#postgresql-nodes): ```ruby # Disable all components except Pgbouncer and Consul agent diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 775d78eb903562..a41d6a77aed44e 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -754,9 +754,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. ``` NOTE: -Postgres with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. +PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](https://docs.gitlab.com/ee/administration/postgresql/replication_and_failover.html#selecting-the-appropriate-patroni-replication-method) for more detailed information. +Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, @@ -827,7 +827,7 @@ The following IPs will be used as an example: 1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`, and replace `` and `` with the - password hashes you [set up previously](#postgresql-primary-node): + password hashes you [set up previously](#postgresql-nodes): ```ruby # Disable all components except Pgbouncer and Consul agent -- GitLab From 0d5f7278df46a88d83556ff2a48ce359481b851a Mon Sep 17 00:00:00 2001 From: Grant Young Date: Wed, 6 Jan 2021 13:17:22 +0000 Subject: [PATCH 4/7] Update troubleshooting for Patroni RA docs --- .../troubleshooting.md | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index cd1a2bc91dfe34..39eb559eb5b155 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -514,39 +514,24 @@ See the suggested fix [in Geo documentation](../geo/replication/troubleshooting. See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). -## Troubleshooting PostgreSQL +## Troubleshooting PostgreSQL with Patroni -In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs: +In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs for Postgres (which is run through Patroni): ```shell -sudo gitlab-ctl tail postgresql +sudo gitlab-ctl tail patroni ``` -### Consul and PostgreSQL changes not taking effect +### Consul and PostgreSQL with Patroni changes not taking effect Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. -To restart either service, run `gitlab-ctl restart SERVICE` +To restart either service, run `gitlab-ctl restart consul` or `gitlab-ctl restart patroni` respectively. -For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. +For PostgreSQL with Patroni, to prevent the primary node from being failed over automatically, it's safest to stop all secondaries first, then restart the primary and finally restart the secondaries again. On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../consul.md#restart-consul) for instructions on how to restart the service. -### `gitlab-ctl repmgr-check-master` command produces errors - -If this command displays errors about database permissions it is likely that something failed during -install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these -steps to fix the problem: - -1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` -1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` -1. Exit the database prompt - `\q` -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) and the user will be re-added with the proper permissions. -1. Change to the `gitlab-consul` user - `su - gitlab-consul` -1. Try the check command again - `gitlab-ctl repmgr-check-master`. - -Now there should not be errors. If errors still occur then there is another problem. - ### PgBouncer error `ERROR: pgbouncer cannot connect to server` You may get this error when running `gitlab-rake gitlab:db:configure` or you -- GitLab From c4225f37c8ed9ab3c39aad25f96de19569e83685 Mon Sep 17 00:00:00 2001 From: Grant Young Date: Wed, 6 Jan 2021 13:28:05 +0000 Subject: [PATCH 5/7] Lint fix in RA Troubleshooting docs --- doc/administration/reference_architectures/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index 39eb559eb5b155..8fcdb457d6a8ea 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -516,7 +516,7 @@ See the suggested fix [in Geo documentation](../geo/replication/troubleshooting. ## Troubleshooting PostgreSQL with Patroni -In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs for Postgres (which is run through Patroni): +In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs for PostgreSQL (which is run through Patroni): ```shell sudo gitlab-ctl tail patroni -- GitLab From 2c26b0943697a5d6a54810b03772ba461abc7e1b Mon Sep 17 00:00:00 2001 From: Grant Young Date: Wed, 6 Jan 2021 17:19:57 +0000 Subject: [PATCH 6/7] Update PgBouncer config for RA docs --- .../reference_architectures/10k_users.md | 15 ++++++++------- .../reference_architectures/25k_users.md | 15 ++++++++------- .../reference_architectures/3k_users.md | 15 ++++++++------- .../reference_architectures/50k_users.md | 15 ++++++++------- .../reference_architectures/5k_users.md | 15 ++++++++------- 5 files changed, 40 insertions(+), 35 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 930dbf4d960555..b19714300e2f6c 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -586,15 +586,16 @@ The following IPs will be used as an example: # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - pgbouncer['users'] = { - 'gitlab-consul': { - password: '' - }, - 'pgbouncer': { - password: '' - } + 'gitlab-consul': { + password: '' + }, + 'pgbouncer': { + password: '' + } } + # Incoming recommended value for max db connections is 150. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + pgbouncer['max_db_connections'] = 150 # Configure Consul agent consul['watchers'] = %w(postgresql) diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 5c7003facd5383..ebe3585999c034 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -586,15 +586,16 @@ The following IPs will be used as an example: # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - pgbouncer['users'] = { - 'gitlab-consul': { - password: '' - }, - 'pgbouncer': { - password: '' - } + 'gitlab-consul': { + password: '' + }, + 'pgbouncer': { + password: '' + } } + # Incoming recommended value for max db connections is 150. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + pgbouncer['max_db_connections'] = 150 # Configure Consul agent consul['watchers'] = %w(postgresql) diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 6a04b42771450d..db7c326b966146 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -837,15 +837,16 @@ The following IPs will be used as an example: # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - pgbouncer['users'] = { - 'gitlab-consul': { - password: '' - }, - 'pgbouncer': { - password: '' - } + 'gitlab-consul': { + password: '' + }, + 'pgbouncer': { + password: '' + } } + # Incoming recommended value for max db connections is 150. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + pgbouncer['max_db_connections'] = 150 # Configure Consul agent consul['watchers'] = %w(postgresql) diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index c2f75db417b150..b3cff32d465292 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -586,15 +586,16 @@ The following IPs will be used as an example: # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - pgbouncer['users'] = { - 'gitlab-consul': { - password: '' - }, - 'pgbouncer': { - password: '' - } + 'gitlab-consul': { + password: '' + }, + 'pgbouncer': { + password: '' + } } + # Incoming recommended value for max db connections is 150. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + pgbouncer['max_db_connections'] = 150 # Configure Consul agent consul['watchers'] = %w(postgresql) diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index a41d6a77aed44e..156f3dbd8a8de6 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -835,15 +835,16 @@ The following IPs will be used as an example: # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - pgbouncer['users'] = { - 'gitlab-consul': { - password: '' - }, - 'pgbouncer': { - password: '' - } + 'gitlab-consul': { + password: '' + }, + 'pgbouncer': { + password: '' + } } + # Incoming recommended value for max db connections is 150. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. + pgbouncer['max_db_connections'] = 150 # Configure Consul agent consul['watchers'] = %w(postgresql) -- GitLab From 97784ca0323b48c7534c9c93008a8725e30aa93b Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Thu, 7 Jan 2021 19:14:21 +0000 Subject: [PATCH 7/7] Apply 10 suggestion(s) to 5 file(s) --- doc/administration/reference_architectures/10k_users.md | 9 ++++----- doc/administration/reference_architectures/25k_users.md | 9 ++++----- doc/administration/reference_architectures/3k_users.md | 9 ++++----- doc/administration/reference_architectures/50k_users.md | 9 ++++----- doc/administration/reference_architectures/5k_users.md | 9 ++++----- 5 files changed, 20 insertions(+), 25 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index b19714300e2f6c..afa53b5efa8232 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -504,10 +504,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` -NOTE: -PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. -Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. +PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods, this has a small chance of leading to data loss. +Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index ebe3585999c034..d96e93d4ab4465 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -504,10 +504,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` -NOTE: -PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. -Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. +PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods, this has a small chance of leading to data loss. +Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index db7c326b966146..e7b151452a5274 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -702,7 +702,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -754,10 +754,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` -NOTE: -PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. -Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. +PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods, this has a small chance of leading to data loss. +Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index b3cff32d465292..093869d331b220 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -452,7 +452,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -504,10 +504,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` -NOTE: -PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. -Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. +PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods, this has a small chance of leading to data loss. +Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 156f3dbd8a8de6..16ad866a108201 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -701,7 +701,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 gitlab-consul ``` -1. On the every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: +1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul @@ -753,10 +753,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` -NOTE: -PostgreSQL with Patroni managing it's failover will default to use `pg_rewind` by default to handle conflicts. -Like most failover handling methods this has a small chance of leading to data loss. -Please refer to these [docs](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method) for more detailed information. +PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` by default to handle conflicts. +Like most failover handling methods, this has a small chance of leading to data loss. +Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace the file of the same name on this server. If that file is not on this server, -- GitLab