From f53f501772c880a313a7edefdcd17917277e992c Mon Sep 17 00:00:00 2001 From: Sami Hiltunen Date: Fri, 27 Jan 2023 16:16:23 +0200 Subject: [PATCH 1/2] Update documentation to match Praefect's new configuration structure Omnibus preivously had a custom configuration structure for Praefect that didn't match the actual structure of Praefect. This made a number of things more difficult than they had to be. Omnibus was recently updated to place Praefect's configuration under praefect['configuration'] hash which gets directly marshaled as TOML for the final Praefect configuration. While there is still backwards compatibility handling in place, we should update the documentation to use the new structure in preparation for deprecating the old configuration keys. This commit upgrades Praefect's documentation to refer to the new keys. --- doc/administration/gitaly/praefect.md | 317 ++++++++++++------ doc/administration/gitaly/recovery.md | 66 ++-- doc/administration/gitaly/troubleshooting.md | 8 +- .../reference_architectures/10k_users.md | 103 +++--- .../reference_architectures/25k_users.md | 105 +++--- .../reference_architectures/3k_users.md | 105 +++--- .../reference_architectures/50k_users.md | 105 +++--- .../reference_architectures/5k_users.md | 105 +++--- doc/administration/sidekiq/index.md | 2 +- 9 files changed, 581 insertions(+), 335 deletions(-) diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index abcd26cae1b79f..de719d3ae1e098 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -269,11 +269,16 @@ The database used by Praefect is now configured. You can now configure Praefect to use the database: ```ruby -praefect['database_host'] = POSTGRESQL_HOST -praefect['database_port'] = 5432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' +praefect['configuration'] = { + # ... + database: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + } +} ``` If you see Praefect database errors after configuring PostgreSQL, see @@ -285,19 +290,27 @@ Praefect performance can be improved by additionally configuring the `database_d settings: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432 + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } + } +} ``` When configured, this connection is automatically used for the @@ -322,15 +335,21 @@ To configure PgBouncer in front of the PostgreSQL instance, you must point Praef parameters on Praefect configuration: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' -#praefect['database_sslmode'] = '...' -#praefect['database_sslcert'] = '...' -#praefect['database_sslkey'] = '...' -#praefect['database_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } +} ``` Praefect requires an additional connection to the PostgreSQL that supports the @@ -399,23 +418,30 @@ praefect_production_direct = host=POSTGRESQL_HOST auth_user=pgbouncer dbname=pra Now you can configure Praefect to use PgBouncer for both connections: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -# `PRAEFECT_SQL_PASSWORD` is the plain-text password of -# Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. -praefect['database_password'] = PRAEFECT_SQL_PASSWORD - -praefect['database_dbname'] = 'praefect_production' -praefect['database_direct_dbname'] = 'praefect_production_direct' - -# There is no need to repeat the following. Parameters of direct -# database connection will fall back to the values above. - -#praefect['database_direct_host'] = PGBOUNCER_HOST -#praefect['database_direct_port'] = 6432 -#praefect['database_direct_user'] = 'praefect' -#praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + # `PRAEFECT_SQL_PASSWORD` is the plain-text password of + # Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + session_pooled: { + # ... + dbname: 'praefect_production_direct', + # There is no need to repeat the following. Parameters of direct + # database connection will fall back to the values above. + # + # host: PGBOUNCER_HOST, + # port: 6432, + # user: 'praefect', + # password: PRAEFECT_SQL_PASSWORD, + }, + }, +} ``` With this configuration, Praefect uses PgBouncer for both connection types. @@ -434,19 +460,28 @@ to PostgreSQL. This is the connection that supports the `LISTEN` feature. An example of Praefect configuration that bypasses PgBouncer and directly connects to PostgreSQL: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + # + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + }, + }, +} ``` ### Praefect @@ -501,30 +536,42 @@ Updates to example must be made at: `/etc/gitlab/gitlab.rb`: ```ruby - praefect['listen_addr'] = '0.0.0.0:2305' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + } ``` 1. Configure Prometheus metrics by editing `/etc/gitlab/gitlab.rb`: ```ruby - # Enable Prometheus metrics access to Praefect. You must use firewalls - # to restrict access to this address/port. - # The default metrics endpoint is /metrics - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' - - # Some metrics run queries against the database. Enabling separate database metrics allows - # these metrics to be collected when the metrics are - # scraped on a separate /db_metrics endpoint. - praefect['separate_database_metrics'] = true + praefect['configuration'] = { + # ... + # + # Enable Prometheus metrics access to Praefect. You must use firewalls + # to restrict access to this address/port. + # The default metrics endpoint is /metrics + prometheus_listen_addr: '0.0.0.0:9652', + # Some metrics run queries against the database. Enabling separate database metrics allows + # these metrics to be collected when the metrics are + # scraped on a separate /db_metrics endpoint. + prometheus_exclude_database_from_default_metrics: true, + } ``` -1. Configure a strong `auth_token` for **Praefect** by editing +1. Configure a strong authentication token for **Praefect** by editing `/etc/gitlab/gitlab.rb`. This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster: ```ruby - praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN' + praefect['configuration'] = { + # ... + auth: { + # ... + token: 'PRAEFECT_EXTERNAL_TOKEN', + }, + } ``` 1. Configure **Praefect** to [connect to the PostgreSQL database](#postgresql). We @@ -533,19 +580,32 @@ Updates to example must be made at: If you want to use a TLS client certificate, the options below can be used: ```ruby - # Connect to PostgreSQL using a TLS client certificate - # praefect['database_sslcert'] = '/path/to/client-cert' - # praefect['database_sslkey'] = '/path/to/client-key' - - # Trust a custom certificate authority - # praefect['database_sslrootcert'] = '/path/to/rootcert' + praefect['configuration'] = { + # ... + database: { + # ... + # + # Connect to PostgreSQL using a TLS client certificate + # sslcert: '/path/to/client-cert', + # sslkey: '/path/to/client-key', + # + # Trust a custom certificate authority + # sslrootcert: '/path/to/rootcert', + }, + } ``` By default, Praefect refuses to make an unencrypted connection to PostgreSQL. You can override this by uncommenting the following line: ```ruby - # praefect['database_sslmode'] = 'disable' + praefect['configuration'] = { + # ... + database: { + # ... + # sslmode: 'disable', + }, + } ``` 1. Configure the **Praefect** cluster to connect to each Gitaly node in the @@ -579,23 +639,31 @@ Updates to example must be made at: ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab # server ('default') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://GITALY_HOST_1:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN', - }, - 'gitaly-2' => { - 'address' => 'tcp://GITALY_HOST_2:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://GITALY_HOST_1:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-2', + address: 'tcp://GITALY_HOST_2:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-3', + address: 'tcp://GITALY_HOST_3:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + ], }, - 'gitaly-3' => { - 'address' => 'tcp://GITALY_HOST_3:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - } - } - } + ], } ``` @@ -681,7 +749,14 @@ Note the following: This allows you to do a gradual transition from unencrypted to encrypted traffic, if necessary. - To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + To disable the unencrypted listener, set: + + ```ruby + praefect['configuration'] = { + # ... + listen_addr: nil, + } + ``` To configure Praefect with TLS: @@ -702,9 +777,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -814,11 +895,11 @@ because we rely on Praefect to route operations correctly. Particular attention should be shown to: - The `gitaly['auth_token']` configured in this section must match the `token` - value under `praefect['virtual_storages']['nodes']` on the Praefect node. This was set + value under `praefect['configuration'][:virtual_storage][][:node][][:token]` on the Praefect node. This was set in the [previous section](#praefect). This document uses the placeholder `PRAEFECT_INTERNAL_TOKEN` throughout. - The storage names in `git_data_dirs` configured in this section must match the - storage names under `praefect['virtual_storages']` on the Praefect node. This + storage names under `praefect['configuration'][:virtual_storage]` on the Praefect node. This was set in the [previous section](#praefect). This document uses `gitaly-1`, `gitaly-2`, and `gitaly-3` as Gitaly storage names. @@ -1000,7 +1081,7 @@ application. This is done by updating the `git_data_dirs`. Particular attention should be shown to: - the storage name added to `git_data_dirs` in this section must match the - storage name under `praefect['virtual_storages']` on the Praefect nodes. This + storage name under `praefect['configuration'][:virtual_storage]` on the Praefect nodes. This was set in the [Praefect](#praefect) section of this guide. This document uses `default` as the Praefect storage name. @@ -1219,12 +1300,16 @@ You can configure: The configuration is added to the `/etc/gitlab/gitlab.rb` file: ```ruby - praefect['virtual_storages'] = { - 'default' => { - 'default_replication_factor' => 1, + praefect['configuration'] = { # ... - } - } + virtual_storage: [ + { + # ... + name: 'default', + default_replication_factor: 1, + }, + ], + } ``` - A replication factor for an existing repository using the `set-replication-factor` sub-command. @@ -1313,13 +1398,25 @@ interval is configurable with any valid [Go duration string](https://pkg.go.dev/ To verify the metadata every three days: ```ruby -praefect['background_verification_verification_interval'] = '72h' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '72h', + }, +} ``` Values of 0 and below disable the background verifier. ```ruby -praefect['background_verification_verification_interval'] = '0' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '0', + }, +} ``` #### Enable deletions @@ -1335,7 +1432,13 @@ metrics for them. You can enable deleting invalid metadata records with: ```ruby -praefect['background_verification_delete_invalid_records'] = true +praefect['configuration'] = { + # ... + background_verification: { + # ... + delete_invalid_records: true, + }, +} ``` ### Prioritize verification manually diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 1207d7af3e7f56..32ca0a5281339a 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -277,15 +277,33 @@ The reconciliation frequency can be changed via the configuration. The value can Examples: ```ruby -praefect['reconciliation_scheduling_interval'] = '5m' # the default value +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '5m', # the default value + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '30s' # reconcile every 30 seconds +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '30s', # reconcile every 30 seconds + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '0' # disable the feature +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '0', # disable the feature + }, +} ``` ### Manual reconciliation @@ -334,16 +352,21 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml remove-repository -virtual-storage -repository -apply ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration']['virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` @@ -415,16 +438,21 @@ The `track-repository` Praefect sub-command adds repositories on disk to the Pra sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml track-repository -virtual-storage -authoritative-storage -repository -replicate-immediately ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration'][:virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index df189f979414bc..46672620f2d6b3 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -500,9 +500,9 @@ Here are common errors and potential causes: - 500 response code - `ActionView::Template::Error (7:permission denied)` - - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. + - `praefect['configuration'][:auth][:token]` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - `Unable to save project. Error: 7:permission denied` - - Secret token in `praefect['storage_nodes']` on GitLab server does not match the + - Secret token in `praefect['configuration'][:virtual_storage]` on GitLab server does not match the value in `gitaly['auth_token']` on one or more Gitaly servers. - 503 response code - `GRPC::Unavailable (14:failed to connect to all addresses)` @@ -516,7 +516,7 @@ Here are common errors and potential causes: Some common reasons for the Praefect database to experience elevated CPU usage include: - Prometheus metrics scrapes [running an expensive query](https://gitlab.com/gitlab-org/gitaly/-/issues/3796). If you have GitLab 14.2 - or above, set `praefect['separate_database_metrics'] = true` in `gitlab.rb`. + or above, set `praefect['configuration'][:prometheus_exclude_database_from_default_metrics] = true` in `gitlab.rb`. - [Read distribution caching](praefect.md#reads-distribution-caching) is disabled, increasing the number of queries made to the database when user traffic is high. Ensure read distribution caching is enabled. @@ -636,7 +636,7 @@ If the supplied value for `-virtual-storage` is incorrect, the command returns t get metadata: rpc error: code = NotFound desc = repository not found ``` -The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['virtual_storages']` in `/etc/gitlab/gitlab.rb`. +The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['configuration'][:virtual_storage]` in `/etc/gitlab/gitlab.rb`. ### Check that repositories are in sync diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index dcba034b381de8..e4fdba403eb7be 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1389,7 +1389,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1404,45 +1403,63 @@ Updates to example must be made at: # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1646,7 +1663,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS pass-through. Refer to the load balancers documentation on how to configure this. @@ -1668,9 +1685,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 48e6d01a2e57cf..dc23f1ca67f5cd 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1406,7 +1406,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1415,51 +1414,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1663,7 +1680,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1685,9 +1702,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 2f65f3ba2e1c80..9fe8ca3ffda513 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1341,7 +1341,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1350,51 +1349,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1598,7 +1615,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1620,9 +1637,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index e12f40b734eb11..bee01d698f4829 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1402,7 +1402,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1411,51 +1410,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1659,7 +1676,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1681,9 +1698,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 113e9a26a54604..81caf537a9aaf1 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1338,7 +1338,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1347,51 +1346,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1595,7 +1612,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1617,9 +1634,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index 7b3ecdd0890052..315d7a2679dd85 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -37,7 +37,7 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token gitaly['auth_token'] = 'abc123secret' - praefect['auth_token'] = 'abc123secret' + praefect['configuration'][:auth][:token] = 'abc123secret' gitlab_rails['gitaly_token'] = 'abc123secret' ## Redis configuration -- GitLab From 4e2b9002a7b342c4468acbe4001e685604cdc2a9 Mon Sep 17 00:00:00 2001 From: Sami Hiltunen Date: Tue, 7 Feb 2023 12:29:08 +0200 Subject: [PATCH 2/2] Add migration instructions for Praefect's new configuration This commit documents migration steps to Praefect's new configuration structure in the upgrade notes --- doc/update/index.md | 132 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/doc/update/index.md b/doc/update/index.md index ade300e4a64007..b783d746f222bb 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -268,6 +268,138 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap - This version removes `SanitizeConfidentialTodos` background migration which was [added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908/diffs) in 15.6 and removed any user inaccessible Todos. Make sure that this migration is finished before upgrading to 15.9. - As part of the [CI Partitioning effort](../architecture/blueprints/ci_data_decay/pipeline_partitioning.md), a [new Foreign Key](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107547) was added to `ci_builds_needs`. On GitLab instances with large CI tables, adding this constraint can take longer than usual. Make sure that this migration is finished before upgrading to 15.9. +- Praefect configuration structure in Omnibus GitLab [has changed](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) to be consistent with the Praefect configuration structure + used in source installs. Praefect configuration is now under `praefect['configuration']` as a single hash. There are still other top-level keys in `praefect` used by + Omnibus GitLab. + + Migrate by moving your existing configuration under the new structure. Below is the new structure with the old keys described in a comment above the key. Replace the + `...` with the value from the old key. Default values are the same. If you haven't configured a value previously, you don't need to configure it. Remove the old keys + from the configuration once migrated. + + ```ruby + praefect['configuration'] = { + # praefect['listen_addr'] + listen_addr: ..., + # praefect['socket_path'] + socket_path: ..., + # praefect['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # praefect['tls_listen_addr'] + tls_listen_addr: ..., + # praefect['separate_database_metrics'] + prometheus_exclude_database_from_default_metrics: ..., + auth: { + # praefect['auth_token'] + token: ..., + # praefect['auth_transitioning'] + transitioning: ..., + }, + logging: { + # praefect['logging_format'] + format: ..., + # praefect['logging_level'] + level: ..., + }, + failover: { + # praefect['failover_enabled'] + enabled: ..., + }, + background_verification: { + # praefect['background_verification_delete_invalid_records'] + delete_invalid_records: ... + # praefect['background_verification_verification_interval'] + verification_interval: ..., + }, + reconciliation: { + # praefect['reconciliation_scheduling_interval'] + scheduling_interval: ..., + # praefect['reconciliation_histogram_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + histogram_buckets: ..., + }, + tls: { + # praefect['certificate_path'] + certificate_path: ..., + # praefect['key_path'] + key_path: ..., + }, + database: { + # praefect['database_host'] + host: ..., + # praefect['database_port'] + port: ..., + # praefect['database_user'] + user: ..., + # praefect['database_password'] + password: ..., + # praefect['database_dbname'] + dbname: ..., + # praefect['database_sslmode'] + sslmode: ..., + # praefect['database_sslcert'] + sslcert: ..., + # praefect['database_sslkey'] + sslkey: ..., + # praefect['database_sslrootcert'] + sslrootcert: ..., + session_pooled: { + # praefect['database_direct_host'] + host: ..., + # praefect['database_direct_port'] + port: ..., + # praefect['database_direct_user'] + user: ..., + # praefect['database_direct_password'] + password: ..., + # praefect['database_direct_dbname'] + dbname: ..., + # praefect['database_direct_sslmode'] + sslmode: ..., + # praefect['database_direct_sslcert'] + sslcert: ..., + # praefect['database_direct_sslkey'] + sslkey: ..., + # praefect['database_direct_sslrootcert'] + sslrootcert: ..., + } + }, + sentry: { + # praefect['sentry_dsn'] + sentry_dsn: ..., + # praefect['sentry_environment'] + sentry_environment: ..., + }, + prometheus: { + # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + # praefect['graceful_stop_timeout'] + graceful_stop_timeout: ..., + + # praefect['virtual_storages']. The old value was a hash map but the new value is an array. + virtual_storage: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in + # the 'virtual_storages' hash. + name: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map + # but the new value is an array. + node: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the + # storage. + storage: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address']. + address: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token']. + token: ... + }, + ], + } + ] + } + ``` ### 15.8.0 -- GitLab