From cd104229495751c51793860d32cb7584c6f95eba Mon Sep 17 00:00:00 2001 From: Matthias Baur Date: Mon, 5 Feb 2024 09:52:16 +0100 Subject: [PATCH 1/7] Add Rake tasks for automated preflight checks Relates to https://gitlab.com/groups/gitlab-org/-/epics/3279 --- ee/lib/tasks/gitlab/geo.rake | 77 ++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index 7f45219f233f79..38863ec48843bc 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -60,5 +60,82 @@ namespace :gitlab do " gitlab-rake gitlab:geo:check".color(:red) end end + + describe 'Gitlab | Geo | Prevent updates to primary site' + task prevent_updates_to_primary_site: :environment do + if ::Gitlab::Geo.secondary? + abort 'This command is only available on a primary node'.color(:red) + end + + puts 'Put GitLab into maintenance mode' + ::Gitlab::CurrentSettings.update!(maintenance_mode: true) + ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") + + puts 'Sidekiq Queues: Disabling all none Geo queues' + Sidekiq::Cron::Job.all.each { |job| job.disable! } + + # This should be only `geo_sidekiq_cron_config_worker`, but can't due to https://gitlab.com/gitlab-org/gitlab/-/issues/37135 + geo_wanted_jobs = ::Gitlab::Geo::CronManager::COMMON_GEO_JOBS + ::Gitlab::Geo::CronManager::COMMON_GEO_AND_NON_GEO_JOBS + ::Gitlab::Geo::CronManager::PRIMARY_GEO_JOBS + geo_wanted_jobs.each { |name| Sidekiq::Cron::Job.find(name).enable! } + + puts "Sidekiq Queues: Waiting for all none Geo queues to be empty" + until Sidekiq::Queue.all.select { |queue| ! queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? + sleep(1) + end + puts "Sidekiq Queues: None Geo queues empty".color(:green) + end + + describe 'Gitlab | Geo | Wait until replicated and verified' + task wait_until_replicated_and_verified: :environment do + unless ::Gitlab::Geo.secondary? + abort 'This command is only available on a secondary node'.color(:red) + end + + puts "Sidekiq Queues: Waiting for all Geo queues to be empty" + until Sidekiq::Queue.all.select { |queue| queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? + sleep(1) + end + puts "Sidekiq Queues: Geo queues empty".color(:green) + + current_node_status = GeoNodeStatus.current_node_status + geo_node = current_node_status.geo_node + + puts "Database replication: Waiting for replication lag == 0" + until current_node_status.db_replication_lag_seconds == 0 + sleep(1) + end + puts "Database replication: Caught up".color(:green) + + puts "Geo log cursor: Wait for events being equal on primary and secondary" + # last_event_id => Primary, geo_cursor_last_event_id => Secondary + until current_node_status.last_event_id == current_node_status.cursor_last_event_id + puts "Geo log cursor: #{current_node_status.last_event_id} != #{current_node_status.cursor_last_event_id}" + sleep(1) + end + puts "Geo log cursor: Caught up".color(:green) + + puts "Data replication/verification: Wait for all data to be replication and verified" + until Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).replication_verification_complete? + Gitlab::Geo.enabled_replicator_classes.each do |replicator_class| + # Sync State + next unless current_node_status.count_for(replicator_class) > 0 + next unless current_node_status.synced_in_percentage_for(replicator_class) < 100 + + Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: replicator_class.replicable_title_plural, failed: replicator_class.failed_count, succeeded: replicator_class.synced_count, total: replicator_class.registry_count, percentage: current_node_status.synced_in_percentage_for(replicator_class)) + + # Verification State + next unless replicator_class.verification_enabled? + next unless current_node_status.verified_in_percentage_for(replicator_class) < 100 + + Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: "#{replicator_class.replicable_title_plural} Verified", failed: replicator_class.verification_failed_count, succeeded: replicator_class.verified_count, total: replicator_class.registry_count, percentage: current_node_status.verified_in_percentage_for(replicator_class)) + end + + sleep(1) + + # Update status + current_node_status = GeoNodeStatus.current_node_status + end + puts "Data replication/verification: All data successfully replicated and verified".color(:green) + end end end -- GitLab From 7cb17f63c7eaa0cb100e3bcff328af99a2250187 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 6 Feb 2024 08:02:56 +0000 Subject: [PATCH 2/7] Apply 4 suggestion(s) to 1 file(s) --- ee/lib/tasks/gitlab/geo.rake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index 38863ec48843bc..e79cfa671b1ee0 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -71,18 +71,18 @@ namespace :gitlab do ::Gitlab::CurrentSettings.update!(maintenance_mode: true) ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") - puts 'Sidekiq Queues: Disabling all none Geo queues' + puts 'Sidekiq Queues: Disabling all non-Geo queues' Sidekiq::Cron::Job.all.each { |job| job.disable! } - # This should be only `geo_sidekiq_cron_config_worker`, but can't due to https://gitlab.com/gitlab-org/gitlab/-/issues/37135 + # Do not enable `geo_sidekiq_cron_config_worker`, due to https://gitlab.com/gitlab-org/gitlab/-/issues/37135 geo_wanted_jobs = ::Gitlab::Geo::CronManager::COMMON_GEO_JOBS + ::Gitlab::Geo::CronManager::COMMON_GEO_AND_NON_GEO_JOBS + ::Gitlab::Geo::CronManager::PRIMARY_GEO_JOBS geo_wanted_jobs.each { |name| Sidekiq::Cron::Job.find(name).enable! } - puts "Sidekiq Queues: Waiting for all none Geo queues to be empty" + puts "Sidekiq Queues: Waiting for all non-Geo queues to be empty" until Sidekiq::Queue.all.select { |queue| ! queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? sleep(1) end - puts "Sidekiq Queues: None Geo queues empty".color(:green) + puts "Sidekiq Queues: Non-Geo queues empty".color(:green) end describe 'Gitlab | Geo | Wait until replicated and verified' -- GitLab From 7bddc68c12621b4b7b1b556d1e6c40b99ac839ac Mon Sep 17 00:00:00 2001 From: Matthias Baur Date: Tue, 6 Feb 2024 09:07:32 +0100 Subject: [PATCH 3/7] Make maintenance message configurable --- ee/lib/tasks/gitlab/geo.rake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index e79cfa671b1ee0..dfe853325bcf4d 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -67,6 +67,8 @@ namespace :gitlab do abort 'This command is only available on a primary node'.color(:red) end + maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] ? ENV['MAINTENANCE_MESSAGE'] : 'Gitlab Geo Failover' + puts 'Put GitLab into maintenance mode' ::Gitlab::CurrentSettings.update!(maintenance_mode: true) ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") -- GitLab From 3755479a6d304d2ee6e609d312522270f984745a Mon Sep 17 00:00:00 2001 From: Matthias Baur Date: Mon, 4 Mar 2024 15:29:47 +0100 Subject: [PATCH 4/7] Refactor to Gitlab::Geo::GeoTasks methods --- ee/lib/gitlab/geo/geo_tasks.rb | 69 ++++++++++++++++++++++++++++++++ ee/lib/tasks/gitlab/geo.rake | 73 ++-------------------------------- 2 files changed, 73 insertions(+), 69 deletions(-) diff --git a/ee/lib/gitlab/geo/geo_tasks.rb b/ee/lib/gitlab/geo/geo_tasks.rb index 3e6cd99d87264c..6fd73f9a97c629 100644 --- a/ee/lib/gitlab/geo/geo_tasks.rb +++ b/ee/lib/gitlab/geo/geo_tasks.rb @@ -53,6 +53,75 @@ def update_primary_geo_node_url exit 1 end end + + def prevent_updates_to_primary_site + maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] ? ENV['MAINTENANCE_MESSAGE'] : 'Gitlab Geo Failover' + + $stdout.puts 'Put GitLab into maintenance mode' + ::Gitlab::CurrentSettings.update!(maintenance_mode: true) + ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") + + $stdout.puts 'Sidekiq Queues: Disabling all non-Geo queues' + Sidekiq::Cron::Job.all.each { |job| job.disable! } + + # Do not enable `geo_sidekiq_cron_config_worker`, due to https://gitlab.com/gitlab-org/gitlab/-/issues/37135 + geo_wanted_jobs = ::Gitlab::Geo::CronManager::COMMON_GEO_JOBS + ::Gitlab::Geo::CronManager::COMMON_GEO_AND_NON_GEO_JOBS + ::Gitlab::Geo::CronManager::PRIMARY_GEO_JOBS + geo_wanted_jobs.each { |name| Sidekiq::Cron::Job.find(name).enable! } + + $stdout.puts "Sidekiq Queues: Waiting for all non-Geo queues to be empty" + until Sidekiq::Queue.all.select { |queue| ! queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? + sleep(1) + end + $stdout.puts "Sidekiq Queues: Non-Geo queues empty".color(:green) + end + + def wait_until_replicated_and_verified + puts "Sidekiq Queues: Waiting for all Geo queues to be empty" + until Sidekiq::Queue.all.select { |queue| queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? + sleep(1) + end + puts "Sidekiq Queues: Geo queues empty".color(:green) + + current_node_status = GeoNodeStatus.current_node_status + geo_node = current_node_status.geo_node + + puts "Database replication: Waiting for replication lag == 0" + until current_node_status.db_replication_lag_seconds == 0 + sleep(1) + end + puts "Database replication: Caught up".color(:green) + + puts "Geo log cursor: Wait for events being equal on primary and secondary" + # last_event_id => Primary, geo_cursor_last_event_id => Secondary + until current_node_status.last_event_id == current_node_status.cursor_last_event_id + puts "Geo log cursor: #{current_node_status.last_event_id} != #{current_node_status.cursor_last_event_id}" + sleep(1) + end + puts "Geo log cursor: Caught up".color(:green) + + puts "Data replication/verification: Wait for all data to be replication and verified" + until Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).replication_verification_complete? + Gitlab::Geo.enabled_replicator_classes.each do |replicator_class| + # Sync State + next unless current_node_status.count_for(replicator_class) > 0 + next unless current_node_status.synced_in_percentage_for(replicator_class) < 100 + + Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: replicator_class.replicable_title_plural, failed: replicator_class.failed_count, succeeded: replicator_class.synced_count, total: replicator_class.registry_count, percentage: current_node_status.synced_in_percentage_for(replicator_class)) + + # Verification State + next unless replicator_class.verification_enabled? + next unless current_node_status.verified_in_percentage_for(replicator_class) < 100 + + Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: "#{replicator_class.replicable_title_plural} Verified", failed: replicator_class.verification_failed_count, succeeded: replicator_class.verified_count, total: replicator_class.registry_count, percentage: current_node_status.verified_in_percentage_for(replicator_class)) + end + + sleep(1) + + # Update status + current_node_status = GeoNodeStatus.current_node_status + end + puts "Data replication/verification: All data successfully replicated and verified".color(:green) + end end end end diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index dfe853325bcf4d..e0964a6456c9af 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -63,81 +63,16 @@ namespace :gitlab do describe 'Gitlab | Geo | Prevent updates to primary site' task prevent_updates_to_primary_site: :environment do - if ::Gitlab::Geo.secondary? - abort 'This command is only available on a primary node'.color(:red) - end - - maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] ? ENV['MAINTENANCE_MESSAGE'] : 'Gitlab Geo Failover' - - puts 'Put GitLab into maintenance mode' - ::Gitlab::CurrentSettings.update!(maintenance_mode: true) - ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") + abort 'This command is only available on a primary node' if ::Gitlab::Geo.secondary? - puts 'Sidekiq Queues: Disabling all non-Geo queues' - Sidekiq::Cron::Job.all.each { |job| job.disable! } - - # Do not enable `geo_sidekiq_cron_config_worker`, due to https://gitlab.com/gitlab-org/gitlab/-/issues/37135 - geo_wanted_jobs = ::Gitlab::Geo::CronManager::COMMON_GEO_JOBS + ::Gitlab::Geo::CronManager::COMMON_GEO_AND_NON_GEO_JOBS + ::Gitlab::Geo::CronManager::PRIMARY_GEO_JOBS - geo_wanted_jobs.each { |name| Sidekiq::Cron::Job.find(name).enable! } - - puts "Sidekiq Queues: Waiting for all non-Geo queues to be empty" - until Sidekiq::Queue.all.select { |queue| ! queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? - sleep(1) - end - puts "Sidekiq Queues: Non-Geo queues empty".color(:green) + Gitlab::Geo::GeoTasks.prevent_updates_to_primary_site end describe 'Gitlab | Geo | Wait until replicated and verified' task wait_until_replicated_and_verified: :environment do - unless ::Gitlab::Geo.secondary? - abort 'This command is only available on a secondary node'.color(:red) - end - - puts "Sidekiq Queues: Waiting for all Geo queues to be empty" - until Sidekiq::Queue.all.select { |queue| queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? - sleep(1) - end - puts "Sidekiq Queues: Geo queues empty".color(:green) - - current_node_status = GeoNodeStatus.current_node_status - geo_node = current_node_status.geo_node - - puts "Database replication: Waiting for replication lag == 0" - until current_node_status.db_replication_lag_seconds == 0 - sleep(1) - end - puts "Database replication: Caught up".color(:green) - - puts "Geo log cursor: Wait for events being equal on primary and secondary" - # last_event_id => Primary, geo_cursor_last_event_id => Secondary - until current_node_status.last_event_id == current_node_status.cursor_last_event_id - puts "Geo log cursor: #{current_node_status.last_event_id} != #{current_node_status.cursor_last_event_id}" - sleep(1) - end - puts "Geo log cursor: Caught up".color(:green) + abort 'This command is only available on a secondary node' unless ::Gitlab::Geo.secondary? - puts "Data replication/verification: Wait for all data to be replication and verified" - until Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).replication_verification_complete? - Gitlab::Geo.enabled_replicator_classes.each do |replicator_class| - # Sync State - next unless current_node_status.count_for(replicator_class) > 0 - next unless current_node_status.synced_in_percentage_for(replicator_class) < 100 - - Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: replicator_class.replicable_title_plural, failed: replicator_class.failed_count, succeeded: replicator_class.synced_count, total: replicator_class.registry_count, percentage: current_node_status.synced_in_percentage_for(replicator_class)) - - # Verification State - next unless replicator_class.verification_enabled? - next unless current_node_status.verified_in_percentage_for(replicator_class) < 100 - - Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).send(:print_counts_row, description: "#{replicator_class.replicable_title_plural} Verified", failed: replicator_class.verification_failed_count, succeeded: replicator_class.verified_count, total: replicator_class.registry_count, percentage: current_node_status.verified_in_percentage_for(replicator_class)) - end - - sleep(1) - - # Update status - current_node_status = GeoNodeStatus.current_node_status - end - puts "Data replication/verification: All data successfully replicated and verified".color(:green) + Gitlab::Geo::GeoTasks.wait_until_replicated_and_verified end end end -- GitLab From eb008f4fe759dac6d99b2a24d787f10af2962b80 Mon Sep 17 00:00:00 2001 From: Matthias Baur Date: Mon, 4 Mar 2024 15:45:43 +0100 Subject: [PATCH 5/7] Refactor to multiple methods --- ee/lib/gitlab/geo/geo_tasks.rb | 26 +++++++++++++++++++++----- ee/lib/tasks/gitlab/geo.rake | 3 ++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ee/lib/gitlab/geo/geo_tasks.rb b/ee/lib/gitlab/geo/geo_tasks.rb index 6fd73f9a97c629..49aaa51b80ed26 100644 --- a/ee/lib/gitlab/geo/geo_tasks.rb +++ b/ee/lib/gitlab/geo/geo_tasks.rb @@ -54,13 +54,15 @@ def update_primary_geo_node_url end end - def prevent_updates_to_primary_site + def enable_maintenance_mode maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] ? ENV['MAINTENANCE_MESSAGE'] : 'Gitlab Geo Failover' $stdout.puts 'Put GitLab into maintenance mode' ::Gitlab::CurrentSettings.update!(maintenance_mode: true) ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") + end + def wait_for_empty_non_geo_queues $stdout.puts 'Sidekiq Queues: Disabling all non-Geo queues' Sidekiq::Cron::Job.all.each { |job| job.disable! } @@ -75,22 +77,23 @@ def prevent_updates_to_primary_site $stdout.puts "Sidekiq Queues: Non-Geo queues empty".color(:green) end - def wait_until_replicated_and_verified + def wait_empty_geo_queues puts "Sidekiq Queues: Waiting for all Geo queues to be empty" until Sidekiq::Queue.all.select { |queue| queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? sleep(1) end puts "Sidekiq Queues: Geo queues empty".color(:green) + end - current_node_status = GeoNodeStatus.current_node_status - geo_node = current_node_status.geo_node - + def wait_for_database_replication(current_node_status) puts "Database replication: Waiting for replication lag == 0" until current_node_status.db_replication_lag_seconds == 0 sleep(1) end puts "Database replication: Caught up".color(:green) + end + def wait_for_geo_log_cursor(current_node_status) puts "Geo log cursor: Wait for events being equal on primary and secondary" # last_event_id => Primary, geo_cursor_last_event_id => Secondary until current_node_status.last_event_id == current_node_status.cursor_last_event_id @@ -98,7 +101,9 @@ def wait_until_replicated_and_verified sleep(1) end puts "Geo log cursor: Caught up".color(:green) + end + def wait_for_data_replication_and_verification(current_node_status, geo_node) puts "Data replication/verification: Wait for all data to be replication and verified" until Gitlab::Geo::GeoNodeStatusCheck.new(current_node_status, geo_node).replication_verification_complete? Gitlab::Geo.enabled_replicator_classes.each do |replicator_class| @@ -122,6 +127,17 @@ def wait_until_replicated_and_verified end puts "Data replication/verification: All data successfully replicated and verified".color(:green) end + + def wait_until_replicated_and_verified + wait_empty_geo_queues + + current_node_status = GeoNodeStatus.current_node_status + geo_node = current_node_status.geo_node + + wait_for_database_replication(current_node_status) + wait_for_geo_log_cursor(current_node_status) + wait_for_data_replication_and_verification(current_node_status, geo_node) + end end end end diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index e0964a6456c9af..a942b0eb095394 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -65,7 +65,8 @@ namespace :gitlab do task prevent_updates_to_primary_site: :environment do abort 'This command is only available on a primary node' if ::Gitlab::Geo.secondary? - Gitlab::Geo::GeoTasks.prevent_updates_to_primary_site + Gitlab::Geo::GeoTasks.enable_maintenance_mode + Gitlab::Geo::GeoTasks.wait_for_empty_non_geo_queues end describe 'Gitlab | Geo | Wait until replicated and verified' -- GitLab From 7b52730daec863aa3fbcf57366552f3d4483b80c Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Mar 2024 07:24:05 +0000 Subject: [PATCH 6/7] Apply 1 suggestion(s) to 1 file(s) --- ee/lib/tasks/gitlab/geo.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/lib/tasks/gitlab/geo.rake b/ee/lib/tasks/gitlab/geo.rake index a942b0eb095394..cf2262160bdb7a 100644 --- a/ee/lib/tasks/gitlab/geo.rake +++ b/ee/lib/tasks/gitlab/geo.rake @@ -63,7 +63,7 @@ namespace :gitlab do describe 'Gitlab | Geo | Prevent updates to primary site' task prevent_updates_to_primary_site: :environment do - abort 'This command is only available on a primary node' if ::Gitlab::Geo.secondary? + abort 'This command is only available on a primary node' unless ::Gitlab::Geo.primary? Gitlab::Geo::GeoTasks.enable_maintenance_mode Gitlab::Geo::GeoTasks.wait_for_empty_non_geo_queues -- GitLab From 207e3deaa1411070ec214e45c929e0a42f1ae1c9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 7 Mar 2024 08:08:29 +0000 Subject: [PATCH 7/7] Apply 2 suggestion(s) to 1 file(s) --- ee/lib/gitlab/geo/geo_tasks.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ee/lib/gitlab/geo/geo_tasks.rb b/ee/lib/gitlab/geo/geo_tasks.rb index 49aaa51b80ed26..31fc8fddb8c902 100644 --- a/ee/lib/gitlab/geo/geo_tasks.rb +++ b/ee/lib/gitlab/geo/geo_tasks.rb @@ -55,11 +55,12 @@ def update_primary_geo_node_url end def enable_maintenance_mode - maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] ? ENV['MAINTENANCE_MESSAGE'] : 'Gitlab Geo Failover' + maintenance_mode_message = ENV['MAINTENANCE_MESSAGE'] - $stdout.puts 'Put GitLab into maintenance mode' - ::Gitlab::CurrentSettings.update!(maintenance_mode: true) - ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "Gitlab Geo Failover") + $stdout.puts 'Enabling GitLab Maintenance Mode' + update_attrs = { maintenance_mode: true } + update_attrs.merge!(maintenance_mode_message: maintenance_mode_message) if maintenance_mode_message.present? + ::Gitlab::CurrentSettings.update!(update_attrs) end def wait_for_empty_non_geo_queues @@ -71,7 +72,7 @@ def wait_for_empty_non_geo_queues geo_wanted_jobs.each { |name| Sidekiq::Cron::Job.find(name).enable! } $stdout.puts "Sidekiq Queues: Waiting for all non-Geo queues to be empty" - until Sidekiq::Queue.all.select { |queue| ! queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? + until Sidekiq::Queue.all.select { |queue| !queue.name.include?('geo') && Sidekiq::Queue.new(queue.name).size > 0 }.empty? sleep(1) end $stdout.puts "Sidekiq Queues: Non-Geo queues empty".color(:green) -- GitLab