diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index abcd26cae1b79f610de2b659586341a5336bbea3..de719d3ae1e09878586bc5d33aeef3d14b61abfa 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -269,11 +269,16 @@ The database used by Praefect is now configured. You can now configure Praefect to use the database: ```ruby -praefect['database_host'] = POSTGRESQL_HOST -praefect['database_port'] = 5432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' +praefect['configuration'] = { + # ... + database: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + } +} ``` If you see Praefect database errors after configuring PostgreSQL, see @@ -285,19 +290,27 @@ Praefect performance can be improved by additionally configuring the `database_d settings: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432 + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } + } +} ``` When configured, this connection is automatically used for the @@ -322,15 +335,21 @@ To configure PgBouncer in front of the PostgreSQL instance, you must point Praef parameters on Praefect configuration: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' -#praefect['database_sslmode'] = '...' -#praefect['database_sslcert'] = '...' -#praefect['database_sslkey'] = '...' -#praefect['database_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } +} ``` Praefect requires an additional connection to the PostgreSQL that supports the @@ -399,23 +418,30 @@ praefect_production_direct = host=POSTGRESQL_HOST auth_user=pgbouncer dbname=pra Now you can configure Praefect to use PgBouncer for both connections: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -# `PRAEFECT_SQL_PASSWORD` is the plain-text password of -# Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. -praefect['database_password'] = PRAEFECT_SQL_PASSWORD - -praefect['database_dbname'] = 'praefect_production' -praefect['database_direct_dbname'] = 'praefect_production_direct' - -# There is no need to repeat the following. Parameters of direct -# database connection will fall back to the values above. - -#praefect['database_direct_host'] = PGBOUNCER_HOST -#praefect['database_direct_port'] = 6432 -#praefect['database_direct_user'] = 'praefect' -#praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + # `PRAEFECT_SQL_PASSWORD` is the plain-text password of + # Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + session_pooled: { + # ... + dbname: 'praefect_production_direct', + # There is no need to repeat the following. Parameters of direct + # database connection will fall back to the values above. + # + # host: PGBOUNCER_HOST, + # port: 6432, + # user: 'praefect', + # password: PRAEFECT_SQL_PASSWORD, + }, + }, +} ``` With this configuration, Praefect uses PgBouncer for both connection types. @@ -434,19 +460,28 @@ to PostgreSQL. This is the connection that supports the `LISTEN` feature. An example of Praefect configuration that bypasses PgBouncer and directly connects to PostgreSQL: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + # + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + }, + }, +} ``` ### Praefect @@ -501,30 +536,42 @@ Updates to example must be made at: `/etc/gitlab/gitlab.rb`: ```ruby - praefect['listen_addr'] = '0.0.0.0:2305' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + } ``` 1. Configure Prometheus metrics by editing `/etc/gitlab/gitlab.rb`: ```ruby - # Enable Prometheus metrics access to Praefect. You must use firewalls - # to restrict access to this address/port. - # The default metrics endpoint is /metrics - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' - - # Some metrics run queries against the database. Enabling separate database metrics allows - # these metrics to be collected when the metrics are - # scraped on a separate /db_metrics endpoint. - praefect['separate_database_metrics'] = true + praefect['configuration'] = { + # ... + # + # Enable Prometheus metrics access to Praefect. You must use firewalls + # to restrict access to this address/port. + # The default metrics endpoint is /metrics + prometheus_listen_addr: '0.0.0.0:9652', + # Some metrics run queries against the database. Enabling separate database metrics allows + # these metrics to be collected when the metrics are + # scraped on a separate /db_metrics endpoint. + prometheus_exclude_database_from_default_metrics: true, + } ``` -1. Configure a strong `auth_token` for **Praefect** by editing +1. Configure a strong authentication token for **Praefect** by editing `/etc/gitlab/gitlab.rb`. This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster: ```ruby - praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN' + praefect['configuration'] = { + # ... + auth: { + # ... + token: 'PRAEFECT_EXTERNAL_TOKEN', + }, + } ``` 1. Configure **Praefect** to [connect to the PostgreSQL database](#postgresql). We @@ -533,19 +580,32 @@ Updates to example must be made at: If you want to use a TLS client certificate, the options below can be used: ```ruby - # Connect to PostgreSQL using a TLS client certificate - # praefect['database_sslcert'] = '/path/to/client-cert' - # praefect['database_sslkey'] = '/path/to/client-key' - - # Trust a custom certificate authority - # praefect['database_sslrootcert'] = '/path/to/rootcert' + praefect['configuration'] = { + # ... + database: { + # ... + # + # Connect to PostgreSQL using a TLS client certificate + # sslcert: '/path/to/client-cert', + # sslkey: '/path/to/client-key', + # + # Trust a custom certificate authority + # sslrootcert: '/path/to/rootcert', + }, + } ``` By default, Praefect refuses to make an unencrypted connection to PostgreSQL. You can override this by uncommenting the following line: ```ruby - # praefect['database_sslmode'] = 'disable' + praefect['configuration'] = { + # ... + database: { + # ... + # sslmode: 'disable', + }, + } ``` 1. Configure the **Praefect** cluster to connect to each Gitaly node in the @@ -579,23 +639,31 @@ Updates to example must be made at: ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab # server ('default') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://GITALY_HOST_1:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN', - }, - 'gitaly-2' => { - 'address' => 'tcp://GITALY_HOST_2:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://GITALY_HOST_1:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-2', + address: 'tcp://GITALY_HOST_2:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-3', + address: 'tcp://GITALY_HOST_3:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + ], }, - 'gitaly-3' => { - 'address' => 'tcp://GITALY_HOST_3:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - } - } - } + ], } ``` @@ -681,7 +749,14 @@ Note the following: This allows you to do a gradual transition from unencrypted to encrypted traffic, if necessary. - To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + To disable the unencrypted listener, set: + + ```ruby + praefect['configuration'] = { + # ... + listen_addr: nil, + } + ``` To configure Praefect with TLS: @@ -702,9 +777,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -814,11 +895,11 @@ because we rely on Praefect to route operations correctly. Particular attention should be shown to: - The `gitaly['auth_token']` configured in this section must match the `token` - value under `praefect['virtual_storages']['nodes']` on the Praefect node. This was set + value under `praefect['configuration'][:virtual_storage][][:node][][:token]` on the Praefect node. This was set in the [previous section](#praefect). This document uses the placeholder `PRAEFECT_INTERNAL_TOKEN` throughout. - The storage names in `git_data_dirs` configured in this section must match the - storage names under `praefect['virtual_storages']` on the Praefect node. This + storage names under `praefect['configuration'][:virtual_storage]` on the Praefect node. This was set in the [previous section](#praefect). This document uses `gitaly-1`, `gitaly-2`, and `gitaly-3` as Gitaly storage names. @@ -1000,7 +1081,7 @@ application. This is done by updating the `git_data_dirs`. Particular attention should be shown to: - the storage name added to `git_data_dirs` in this section must match the - storage name under `praefect['virtual_storages']` on the Praefect nodes. This + storage name under `praefect['configuration'][:virtual_storage]` on the Praefect nodes. This was set in the [Praefect](#praefect) section of this guide. This document uses `default` as the Praefect storage name. @@ -1219,12 +1300,16 @@ You can configure: The configuration is added to the `/etc/gitlab/gitlab.rb` file: ```ruby - praefect['virtual_storages'] = { - 'default' => { - 'default_replication_factor' => 1, + praefect['configuration'] = { # ... - } - } + virtual_storage: [ + { + # ... + name: 'default', + default_replication_factor: 1, + }, + ], + } ``` - A replication factor for an existing repository using the `set-replication-factor` sub-command. @@ -1313,13 +1398,25 @@ interval is configurable with any valid [Go duration string](https://pkg.go.dev/ To verify the metadata every three days: ```ruby -praefect['background_verification_verification_interval'] = '72h' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '72h', + }, +} ``` Values of 0 and below disable the background verifier. ```ruby -praefect['background_verification_verification_interval'] = '0' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '0', + }, +} ``` #### Enable deletions @@ -1335,7 +1432,13 @@ metrics for them. You can enable deleting invalid metadata records with: ```ruby -praefect['background_verification_delete_invalid_records'] = true +praefect['configuration'] = { + # ... + background_verification: { + # ... + delete_invalid_records: true, + }, +} ``` ### Prioritize verification manually diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 1207d7af3e7f56281c922a0a815d83df95284b22..32ca0a5281339a899f8fbb2f18b7e15115ed191c 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -277,15 +277,33 @@ The reconciliation frequency can be changed via the configuration. The value can Examples: ```ruby -praefect['reconciliation_scheduling_interval'] = '5m' # the default value +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '5m', # the default value + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '30s' # reconcile every 30 seconds +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '30s', # reconcile every 30 seconds + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '0' # disable the feature +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '0', # disable the feature + }, +} ``` ### Manual reconciliation @@ -334,16 +352,21 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml remove-repository -virtual-storage -repository -apply ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration']['virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` @@ -415,16 +438,21 @@ The `track-repository` Praefect sub-command adds repositories on disk to the Pra sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml track-repository -virtual-storage -authoritative-storage -repository -replicate-immediately ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration'][:virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index df189f979414bcf26a3868099b4c3b1a30a0e1bc..46672620f2d6b35be394596026e8ce62104566a1 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -500,9 +500,9 @@ Here are common errors and potential causes: - 500 response code - `ActionView::Template::Error (7:permission denied)` - - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. + - `praefect['configuration'][:auth][:token]` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - `Unable to save project. Error: 7:permission denied` - - Secret token in `praefect['storage_nodes']` on GitLab server does not match the + - Secret token in `praefect['configuration'][:virtual_storage]` on GitLab server does not match the value in `gitaly['auth_token']` on one or more Gitaly servers. - 503 response code - `GRPC::Unavailable (14:failed to connect to all addresses)` @@ -516,7 +516,7 @@ Here are common errors and potential causes: Some common reasons for the Praefect database to experience elevated CPU usage include: - Prometheus metrics scrapes [running an expensive query](https://gitlab.com/gitlab-org/gitaly/-/issues/3796). If you have GitLab 14.2 - or above, set `praefect['separate_database_metrics'] = true` in `gitlab.rb`. + or above, set `praefect['configuration'][:prometheus_exclude_database_from_default_metrics] = true` in `gitlab.rb`. - [Read distribution caching](praefect.md#reads-distribution-caching) is disabled, increasing the number of queries made to the database when user traffic is high. Ensure read distribution caching is enabled. @@ -636,7 +636,7 @@ If the supplied value for `-virtual-storage` is incorrect, the command returns t get metadata: rpc error: code = NotFound desc = repository not found ``` -The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['virtual_storages']` in `/etc/gitlab/gitlab.rb`. +The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['configuration'][:virtual_storage]` in `/etc/gitlab/gitlab.rb`. ### Check that repositories are in sync diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index dcba034b381de87d86d166cc05d054868d15d4cc..e4fdba403eb7be837e031caad2e7d892196ac230 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1389,7 +1389,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1404,45 +1403,63 @@ Updates to example must be made at: # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1646,7 +1663,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS pass-through. Refer to the load balancers documentation on how to configure this. @@ -1668,9 +1685,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 48e6d01a2e57cf111721da794f8f9073c7608908..dc23f1ca67f5cdcb3e7f6318f784fdaf477c7278 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1406,7 +1406,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1415,51 +1414,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1663,7 +1680,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1685,9 +1702,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 2f65f3ba2e1c80b8fd94a1f449b2caecc6764fff..9fe8ca3ffda51301274e74f37de5064129d3a004 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1341,7 +1341,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1350,51 +1349,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1598,7 +1615,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1620,9 +1637,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index e12f40b734eb11ef27863d32f1d70a78be6fd372..bee01d698f48293a2297779bc7fca2c44ad08060 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1402,7 +1402,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1411,51 +1410,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1659,7 +1676,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1681,9 +1698,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 113e9a26a5460428d5ccc1131a1d4b03cc8801d9..81caf537a9aaf1d016e70332dfbd7acf4546f6e6 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1338,7 +1338,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1347,51 +1346,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1595,7 +1612,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1617,9 +1634,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index 7b3ecdd0890052a95c129121e3817745ae3b94d3..315d7a2679dd8506bfd7857c5d07fea279f76ed7 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -37,7 +37,7 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token gitaly['auth_token'] = 'abc123secret' - praefect['auth_token'] = 'abc123secret' + praefect['configuration'][:auth][:token] = 'abc123secret' gitlab_rails['gitaly_token'] = 'abc123secret' ## Redis configuration diff --git a/doc/update/index.md b/doc/update/index.md index ade300e4a6400737240f173b4157926971d05e87..b783d746f222bbb0d0c2d225ad680cd3512202a5 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -268,6 +268,138 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap - This version removes `SanitizeConfidentialTodos` background migration which was [added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908/diffs) in 15.6 and removed any user inaccessible Todos. Make sure that this migration is finished before upgrading to 15.9. - As part of the [CI Partitioning effort](../architecture/blueprints/ci_data_decay/pipeline_partitioning.md), a [new Foreign Key](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107547) was added to `ci_builds_needs`. On GitLab instances with large CI tables, adding this constraint can take longer than usual. Make sure that this migration is finished before upgrading to 15.9. +- Praefect configuration structure in Omnibus GitLab [has changed](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) to be consistent with the Praefect configuration structure + used in source installs. Praefect configuration is now under `praefect['configuration']` as a single hash. There are still other top-level keys in `praefect` used by + Omnibus GitLab. + + Migrate by moving your existing configuration under the new structure. Below is the new structure with the old keys described in a comment above the key. Replace the + `...` with the value from the old key. Default values are the same. If you haven't configured a value previously, you don't need to configure it. Remove the old keys + from the configuration once migrated. + + ```ruby + praefect['configuration'] = { + # praefect['listen_addr'] + listen_addr: ..., + # praefect['socket_path'] + socket_path: ..., + # praefect['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # praefect['tls_listen_addr'] + tls_listen_addr: ..., + # praefect['separate_database_metrics'] + prometheus_exclude_database_from_default_metrics: ..., + auth: { + # praefect['auth_token'] + token: ..., + # praefect['auth_transitioning'] + transitioning: ..., + }, + logging: { + # praefect['logging_format'] + format: ..., + # praefect['logging_level'] + level: ..., + }, + failover: { + # praefect['failover_enabled'] + enabled: ..., + }, + background_verification: { + # praefect['background_verification_delete_invalid_records'] + delete_invalid_records: ... + # praefect['background_verification_verification_interval'] + verification_interval: ..., + }, + reconciliation: { + # praefect['reconciliation_scheduling_interval'] + scheduling_interval: ..., + # praefect['reconciliation_histogram_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + histogram_buckets: ..., + }, + tls: { + # praefect['certificate_path'] + certificate_path: ..., + # praefect['key_path'] + key_path: ..., + }, + database: { + # praefect['database_host'] + host: ..., + # praefect['database_port'] + port: ..., + # praefect['database_user'] + user: ..., + # praefect['database_password'] + password: ..., + # praefect['database_dbname'] + dbname: ..., + # praefect['database_sslmode'] + sslmode: ..., + # praefect['database_sslcert'] + sslcert: ..., + # praefect['database_sslkey'] + sslkey: ..., + # praefect['database_sslrootcert'] + sslrootcert: ..., + session_pooled: { + # praefect['database_direct_host'] + host: ..., + # praefect['database_direct_port'] + port: ..., + # praefect['database_direct_user'] + user: ..., + # praefect['database_direct_password'] + password: ..., + # praefect['database_direct_dbname'] + dbname: ..., + # praefect['database_direct_sslmode'] + sslmode: ..., + # praefect['database_direct_sslcert'] + sslcert: ..., + # praefect['database_direct_sslkey'] + sslkey: ..., + # praefect['database_direct_sslrootcert'] + sslrootcert: ..., + } + }, + sentry: { + # praefect['sentry_dsn'] + sentry_dsn: ..., + # praefect['sentry_environment'] + sentry_environment: ..., + }, + prometheus: { + # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + # praefect['graceful_stop_timeout'] + graceful_stop_timeout: ..., + + # praefect['virtual_storages']. The old value was a hash map but the new value is an array. + virtual_storage: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in + # the 'virtual_storages' hash. + name: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map + # but the new value is an array. + node: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the + # storage. + storage: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address']. + address: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token']. + token: ... + }, + ], + } + ] + } + ``` ### 15.8.0