From 9ef001a28b73071cd59a285835dd0a19c9f3bc04 Mon Sep 17 00:00:00 2001 From: Grant Young Date: Thu, 15 Apr 2021 11:33:59 +0100 Subject: [PATCH 1/4] Update Sidekiq Cluster guidance in RA docs Guidance around worker counts and tuning --- .../reference_architectures/10k_users.md | 15 +++++++++++++-- .../reference_architectures/25k_users.md | 15 +++++++++++++-- .../reference_architectures/3k_users.md | 12 +++++++++++- .../reference_architectures/50k_users.md | 15 +++++++++++++-- .../reference_architectures/5k_users.md | 14 +++++++++++++- 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 97af1fe8d3c3be..b995fe180dde72 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1917,7 +1917,16 @@ To configure the Sidekiq nodes, on each one: ### Sidekiq configuration ### ####################################### sidekiq['listen_address'] = "0.0.0.0" - sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + # Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = [ + "*", + "*", + "*", + "*" + ] + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 ####################################### ### Monitoring configuration ### @@ -1962,7 +1971,9 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +If found that the environment's Sidekiq job processing is slow with long queues +more nodes can be added as required. You can also tune your Sidekiq nodes to +run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md).
diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 7f9f284d085a17..d66f6cfff04737 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1913,7 +1913,16 @@ To configure the Sidekiq nodes, on each one: ### Sidekiq configuration ### ####################################### sidekiq['listen_address'] = "0.0.0.0" - sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + # Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = [ + "*", + "*", + "*", + "*" + ] + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 ####################################### ### Monitoring configuration ### @@ -1958,7 +1967,9 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +If found that the environment's Sidekiq job processing is slow with long queues +more nodes can be added as required. You can also tune your Sidekiq nodes to +run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md).
diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index b8d0a98a1f1c73..f173decaeb7136 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1595,6 +1595,14 @@ To configure the Sidekiq nodes, one each one: ####################################### sidekiq['listen_address'] = "0.0.0.0" + # Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = [ + "*", + "*" + ] + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 + ####################################### ### Monitoring configuration ### ####################################### @@ -1650,7 +1658,9 @@ To configure the Sidekiq nodes, one each one: ``` NOTE: -You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +If found that the environment's Sidekiq job processing is slow with long queues +more nodes can be added as required. You can also tune your Sidekiq nodes to +run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md).
diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 183a998e89a2f6..2960701457dec6 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1920,7 +1920,16 @@ To configure the Sidekiq nodes, on each one: ### Sidekiq configuration ### ####################################### sidekiq['listen_address'] = "0.0.0.0" - sidekiq['cluster'] = true # no need to set this after GitLab 13.0 + + # Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = [ + "*", + "*", + "*", + "*" + ] + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 ####################################### ### Monitoring configuration ### @@ -1965,7 +1974,9 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +If found that the environment's Sidekiq job processing is slow with long queues +more nodes can be added as required. You can also tune your Sidekiq nodes to +run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md).
diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 70d02bba855e83..9220bcf24c2646 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1585,6 +1585,16 @@ To configure the Sidekiq nodes, one each one: ####################################### sidekiq['listen_address'] = "0.0.0.0" + # Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = [ + "*", + "*", + "*", + "*" + ] + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 + ####################################### ### Monitoring configuration ### ####################################### @@ -1639,7 +1649,9 @@ To configure the Sidekiq nodes, one each one: ``` NOTE: -You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +If found that the environment's Sidekiq job processing is slow with long queues +more nodes can be added as required. You can also tune your Sidekiq nodes to +run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md).
-- GitLab From 8ffc058454e9d870e63eedb1756380481de717ea Mon Sep 17 00:00:00 2001 From: Grant Young Date: Thu, 15 Apr 2021 11:43:03 +0100 Subject: [PATCH 2/4] Update 2k guidance as well --- doc/administration/reference_architectures/2k_users.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index e5418e47ca2974..1631ed946b3bcb 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -642,6 +642,8 @@ On each node perform the following: node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' sidekiq['listen_address'] = "0.0.0.0" + # Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 puma['listen'] = '0.0.0.0' # Add the monitoring node's IP address to the monitoring whitelist and allow it to -- GitLab From c3291f74b7420125ed2a01e45829fc22f182d142 Mon Sep 17 00:00:00 2001 From: Grant Young Date: Mon, 19 Apr 2021 10:32:07 +0100 Subject: [PATCH 3/4] Switch to clearer Sidekiq config --- doc/administration/reference_architectures/10k_users.md | 8 ++------ doc/administration/reference_architectures/25k_users.md | 8 ++------ doc/administration/reference_architectures/3k_users.md | 6 ++---- doc/administration/reference_architectures/50k_users.md | 8 ++------ doc/administration/reference_architectures/5k_users.md | 8 ++------ 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index b995fe180dde72..febcb774b4545a 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1919,12 +1919,8 @@ To configure the Sidekiq nodes, on each one: sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = [ - "*", - "*", - "*", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 4 + # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d66f6cfff04737..fc89e9bd4c820a 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1915,12 +1915,8 @@ To configure the Sidekiq nodes, on each one: sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = [ - "*", - "*", - "*", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 4 + # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index f173decaeb7136..5c0372d76e4cce 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1596,10 +1596,8 @@ To configure the Sidekiq nodes, one each one: sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = [ - "*", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 2 + # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 2960701457dec6..01506e29325128 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1922,12 +1922,8 @@ To configure the Sidekiq nodes, on each one: sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = [ - "*", - "*", - "*", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 4 + # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 9220bcf24c2646..41fd0ac64653d3 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1586,12 +1586,8 @@ To configure the Sidekiq nodes, one each one: sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = [ - "*", - "*", - "*", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 4 + # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 -- GitLab From d9f4e8193563e3a5e04aa6bccd5f618c46d5284e Mon Sep 17 00:00:00 2001 From: Grant Young Date: Wed, 21 Apr 2021 09:47:18 +0000 Subject: [PATCH 4/4] Apply 5 suggestion(s) to 5 file(s) --- doc/administration/reference_architectures/10k_users.md | 2 +- doc/administration/reference_architectures/25k_users.md | 2 +- doc/administration/reference_architectures/3k_users.md | 2 +- doc/administration/reference_architectures/50k_users.md | 2 +- doc/administration/reference_architectures/5k_users.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index febcb774b4545a..177eea6696779d 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1967,7 +1967,7 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -If found that the environment's Sidekiq job processing is slow with long queues +If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index fc89e9bd4c820a..bcdf2a198df437 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1963,7 +1963,7 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -If found that the environment's Sidekiq job processing is slow with long queues +If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 5c0372d76e4cce..b26f4e0ccbacb7 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1656,7 +1656,7 @@ To configure the Sidekiq nodes, one each one: ``` NOTE: -If found that the environment's Sidekiq job processing is slow with long queues +If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 01506e29325128..08f249561f296c 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1970,7 +1970,7 @@ To configure the Sidekiq nodes, on each one: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. NOTE: -If found that the environment's Sidekiq job processing is slow with long queues +If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 41fd0ac64653d3..cb74abe567a4fd 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1645,7 +1645,7 @@ To configure the Sidekiq nodes, one each one: ``` NOTE: -If found that the environment's Sidekiq job processing is slow with long queues +If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). -- GitLab