From 2beb155705d7b76131e7697d940ec3fd59b0a7b1 Mon Sep 17 00:00:00 2001 From: Aleksei Lipniagov Date: Tue, 18 May 2021 17:38:11 +0300 Subject: [PATCH 1/3] Remove Unicorn support: comments Changes no-op code (comments) related to Unicorn removal. Split from https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61888. --- app/models/concerns/prometheus_adapter.rb | 2 +- config/gitlab.yml.example | 10 +++++----- config/initializers/macos.rb | 2 +- config/initializers/rack_timeout.rb | 1 - config/initializers/tracing.rb | 2 +- lib/gitlab/cluster/lifecycle_events.rb | 19 +++++++------------ .../cluster/puma_worker_killer_initializer.rb | 5 ++--- lib/gitlab/stack_prof.rb | 1 - lib/support/init.d/gitlab.default.example | 5 +---- lib/tasks/gitlab/setup.rake | 2 +- workhorse/internal/api/api.go | 6 +++--- 11 files changed, 22 insertions(+), 33 deletions(-) diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index afebc426762241..86280097d192ee 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -38,7 +38,7 @@ def configured? # This is a heavy-weight check if a prometheus is properly configured and accessible from GitLab. # This actually sends a request to an external service and often it could take a long time, - # Please consider using `configured?` instead if the process is running on unicorn/puma threads. + # Please consider using `configured?` instead if the process is running on Puma threads. def can_query? prometheus_client.present? end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index aee9dd455af1cd..e8ad8bec428335 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -33,7 +33,7 @@ production: &base host: localhost port: 80 # Set to 443 if using HTTPS, see installation.md#using-https for additional HTTPS configuration details https: false # Set to true if using HTTPS, see installation.md#using-https for additional HTTPS configuration details - # The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout. + # The maximum time Puma can spend on the request. This needs to be smaller than the worker timeout. # Default is 95% of the worker timeout max_request_duration_seconds: 57 @@ -153,7 +153,7 @@ production: &base ### GraphQL Settings # Tells the rails application how long it has to complete a GraphQL request. # We suggest this value to be higher than the database timeout value - # and lower than the worker timeout set in unicorn/puma. (default: 30) + # and lower than the worker timeout set in Puma. (default: 30) # graphql_timeout: 30 ## Repository downloads directory @@ -1212,8 +1212,8 @@ production: &base ## Monitoring # Built in monitoring settings monitoring: - # Time between sampling of unicorn socket metrics, in seconds - # unicorn_sampler_interval: 10 + # Time between sampling of puma socket metrics, in seconds + # puma_sampler_interval: 10 # IP whitelist to access monitoring endpoints ip_whitelist: - 127.0.0.0/8 @@ -1225,7 +1225,7 @@ production: &base # address: localhost # port: 8082 - # Web exporter is webserver built in to Unicorn/Puma to expose Prometheus metrics + # Web exporter is webserver built in to Puma to expose Prometheus metrics # It runs alongside the `/metrics` endpoints to ease the publish of metrics web_exporter: # enabled: true diff --git a/config/initializers/macos.rb b/config/initializers/macos.rb index f410af6ed4792f..1edd6c0a730c7b 100644 --- a/config/initializers/macos.rb +++ b/config/initializers/macos.rb @@ -5,7 +5,7 @@ require 'fiddle' # Dynamically load Foundation.framework, ~implicitly~ initialising - # the Objective-C runtime before any forking happens in Unicorn + # the Objective-C runtime before any forking happens in webserver # # From https://bugs.ruby-lang.org/issues/14009 Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation' diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index e217398ee7d455..d5027cae08dc18 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# Unicorn terminates any request which runs longer than 60 seconds. # Puma doesn't have any timeout mechanism for terminating long-running # requests, to make sure that server is not paralyzed by long-running # or stuck queries, we add a request timeout which terminates the diff --git a/config/initializers/tracing.rb b/config/initializers/tracing.rb index 3f066953c23534..10b59a9e3d8259 100644 --- a/config/initializers/tracing.rb +++ b/config/initializers/tracing.rb @@ -15,7 +15,7 @@ Labkit::Tracing::Rails::ActionViewSubscriber.instrument Labkit::Tracing::Rails::ActiveSupportSubscriber.instrument - # In multi-processed clustered architectures (puma, unicorn) don't + # In multi-processed clustered architectures (e.g. Puma cluster) don't # start tracing until the worker processes are spawned. This works # around issues when the opentracing implementation spawns threads Gitlab::Cluster::LifecycleEvents.on_worker_start do diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b3dc59466ec0de..6b7b9b09269008 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -8,13 +8,13 @@ module Cluster # LifecycleEvents lets Rails initializers register application startup hooks # that are sensitive to forking. For example, to defer the creation of # watchdog threads. This lets us abstract away the Unix process - # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # lifecycles of Sidekiq, Puma, Puma Cluster, etc. # # We have the following lifecycle events. # # - on_before_fork (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -22,7 +22,7 @@ module Cluster # # - on_master_start (on master process): # - # Unicorn/Puma Cluster: This will be called exactly once, + # Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # @@ -30,7 +30,7 @@ module Cluster # # - on_before_blackout_period (on master process): # - # Unicorn/Puma Cluster: This will be called before a blackout + # Puma Cluster: This will be called before a blackout # period when performing graceful shutdown of master. # This is called on `master` process. # @@ -38,7 +38,7 @@ module Cluster # # - on_before_graceful_shutdown (on master process): # - # Unicorn/Puma Cluster: This will be called before a graceful + # Puma Cluster: This will be called before a graceful # shutdown of workers starts happening, but after blackout period. # This is called on `master` process. # @@ -46,11 +46,6 @@ module Cluster # # - on_before_master_restart (on master process): # - # Unicorn: This will be called before a new master is spun up. - # This is called on forked master before `execve` to become - # a new masterfor Unicorn. This means that this does not really - # affect old master process. - # # Puma Cluster: This will be called before a new master is spun up. # This is called on `master` process. # @@ -58,7 +53,7 @@ module Cluster # # - on_worker_start (on worker process): # - # Unicorn/Puma Cluster: This is called in the worker process + # Puma Cluster: This is called in the worker process # exactly once before processing requests. # # Sidekiq/Puma Single: This is called immediately. @@ -114,7 +109,7 @@ def on_master_start(&block) end # - # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # Lifecycle integration methods (called from puma.rb, etc.) # def do_worker_start call(:worker_start_hooks, @worker_start_hooks) diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index fd9f58a34f36ad..65f76aaad2e19d 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -12,9 +12,8 @@ def self.start( require 'puma_worker_killer' PumaWorkerKiller.config do |config| - # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) - # Importantly RAM is for _all_workers (ie, the cluster), - # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX + # Note! RAM is expressed in megabytes. + # Importantly, RAM is for _all_workers (ie, the cluster), not each worker worker_count = puma_options[:workers] || 1 # The Puma Worker Killer checks the total RAM used by both the master # and worker processes. diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb index 4b7d93c91ce948..97f52491e9ee56 100644 --- a/lib/gitlab/stack_prof.rb +++ b/lib/gitlab/stack_prof.rb @@ -118,7 +118,6 @@ def self.on_worker_start # # see also: # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals - # * https://github.com/phusion/unicorn/blob/master/SIGNALS # * https://github.com/mperham/sidekiq/wiki/Signals Signal.trap('SIGUSR2') do write.write('.') diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1b499467ad6bcd..ef50d718e2f5f6 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -5,9 +5,6 @@ # Normal values are "production", "test" and "development". RAILS_ENV="production" -# Uncomment the line below to enable the Unicorn web server instead of Puma. -# use_web_server="unicorn" - # app_user defines the user that GitLab is run as. # The default is "git". app_user="git" @@ -43,7 +40,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # socket. To listen on TCP connections (needed by Apache) change to: # '-listenNetwork tcp -listenAddr 127.0.0.1:8181' # -# The -authBackend setting tells gitlab-workhorse where it can reach Unicorn. +# The -authBackend setting tells gitlab-workhorse where it can reach Puma. # For relative URL support change to: # '-authBackend http://127.0.0.1/8080/gitlab' # Read more in http://doc.gitlab.com/ce/install/relative_url.html diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 31bd80e78d408b..705519d17417f6 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -40,7 +40,7 @@ namespace :gitlab do end # If there are any clients connected to the DB, PostgreSQL won't let - # you drop the database. It's possible that Sidekiq, Unicorn, or + # you drop the database. It's possible that Sidekiq, Puma, or # some other client will be hanging onto a connection, preventing # the DROP DATABASE from working. To workaround this problem, this # method terminates all the connections so that a subsequent DROP diff --git a/workhorse/internal/api/api.go b/workhorse/internal/api/api.go index d8e2a7b0d9f7b1..129de1fd7b3d3a 100644 --- a/workhorse/internal/api/api.go +++ b/workhorse/internal/api/api.go @@ -308,7 +308,7 @@ func (api *API) PreAuthorizeHandler(next HandleFunc, suffix string) http.Handler return } - httpResponse.Body.Close() // Free up the Unicorn worker + httpResponse.Body.Close() // Free up the Puma thread copyAuthHeader(httpResponse, w) @@ -347,7 +347,7 @@ func copyAuthHeader(httpResponse *http.Response, w http.ResponseWriter) { func passResponseBack(httpResponse *http.Response, w http.ResponseWriter, r *http.Request) { // NGINX response buffering is disabled on this path (with - // X-Accel-Buffering: no) but we still want to free up the Unicorn worker + // X-Accel-Buffering: no) but we still want to free up the Puma thread // that generated httpResponse as fast as possible. To do this we buffer // the entire response body in memory before sending it on. responseBody, err := bufferResponse(httpResponse.Body) @@ -355,7 +355,7 @@ func passResponseBack(httpResponse *http.Response, w http.ResponseWriter, r *htt helper.Fail500(w, r, err) return } - httpResponse.Body.Close() // Free up the Unicorn worker + httpResponse.Body.Close() // Free up the Puma thread bytesTotal.Add(float64(responseBody.Len())) for k, v := range httpResponse.Header { -- GitLab From ed458c6331d5f9441976d3771a4f4c6ce2994166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=A4ppler?= Date: Wed, 19 May 2021 08:34:08 +0000 Subject: [PATCH 2/3] Apply 3 suggestion(s) to 3 file(s) --- config/gitlab.yml.example | 2 +- lib/gitlab/cluster/puma_worker_killer_initializer.rb | 4 ++-- lib/support/init.d/gitlab.default.example | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index e8ad8bec428335..0a0421a60b7f0e 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -1225,7 +1225,7 @@ production: &base # address: localhost # port: 8082 - # Web exporter is webserver built in to Puma to expose Prometheus metrics + # Web exporter is a dedicated Rack server running alongside Puma to expose Prometheus metrics # It runs alongside the `/metrics` endpoints to ease the publish of metrics web_exporter: # enabled: true diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 65f76aaad2e19d..28582ececa7f7d 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -15,8 +15,8 @@ def self.start( # Note! RAM is expressed in megabytes. # Importantly, RAM is for _all_workers (ie, the cluster), not each worker worker_count = puma_options[:workers] || 1 - # The Puma Worker Killer checks the total RAM used by both the master - # and worker processes. + # The Puma Worker Killer checks the total memory used by the cluster, + # i.e. both primary and worker processes. # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 # # Additional memory is added when running in `development` diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index ef50d718e2f5f6..53bebe55fa3697 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -40,7 +40,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # socket. To listen on TCP connections (needed by Apache) change to: # '-listenNetwork tcp -listenAddr 127.0.0.1:8181' # -# The -authBackend setting tells gitlab-workhorse where it can reach Puma. +# The -authBackend setting tells gitlab-workhorse where it can reach the GitLab Rails application. # For relative URL support change to: # '-authBackend http://127.0.0.1/8080/gitlab' # Read more in http://doc.gitlab.com/ce/install/relative_url.html -- GitLab From e9773055aab7823522c96b0953fe12d650ccfa14 Mon Sep 17 00:00:00 2001 From: Aleksei Lipniagov Date: Wed, 19 May 2021 11:35:16 +0300 Subject: [PATCH 3/3] Apply review suggestions --- config/gitlab.yml.example | 10 +++++----- lib/gitlab/cluster/puma_worker_killer_initializer.rb | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 0a0421a60b7f0e..aee9dd455af1cd 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -33,7 +33,7 @@ production: &base host: localhost port: 80 # Set to 443 if using HTTPS, see installation.md#using-https for additional HTTPS configuration details https: false # Set to true if using HTTPS, see installation.md#using-https for additional HTTPS configuration details - # The maximum time Puma can spend on the request. This needs to be smaller than the worker timeout. + # The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout. # Default is 95% of the worker timeout max_request_duration_seconds: 57 @@ -153,7 +153,7 @@ production: &base ### GraphQL Settings # Tells the rails application how long it has to complete a GraphQL request. # We suggest this value to be higher than the database timeout value - # and lower than the worker timeout set in Puma. (default: 30) + # and lower than the worker timeout set in unicorn/puma. (default: 30) # graphql_timeout: 30 ## Repository downloads directory @@ -1212,8 +1212,8 @@ production: &base ## Monitoring # Built in monitoring settings monitoring: - # Time between sampling of puma socket metrics, in seconds - # puma_sampler_interval: 10 + # Time between sampling of unicorn socket metrics, in seconds + # unicorn_sampler_interval: 10 # IP whitelist to access monitoring endpoints ip_whitelist: - 127.0.0.0/8 @@ -1225,7 +1225,7 @@ production: &base # address: localhost # port: 8082 - # Web exporter is a dedicated Rack server running alongside Puma to expose Prometheus metrics + # Web exporter is webserver built in to Unicorn/Puma to expose Prometheus metrics # It runs alongside the `/metrics` endpoints to ease the publish of metrics web_exporter: # enabled: true diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 28582ececa7f7d..e634291f89436a 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -12,8 +12,6 @@ def self.start( require 'puma_worker_killer' PumaWorkerKiller.config do |config| - # Note! RAM is expressed in megabytes. - # Importantly, RAM is for _all_workers (ie, the cluster), not each worker worker_count = puma_options[:workers] || 1 # The Puma Worker Killer checks the total memory used by the cluster, # i.e. both primary and worker processes. -- GitLab