diff --git a/.markdownlint.yml b/.markdownlint.yml index ccf2ef02eee75830268a7ffd37b2b522dc1f0e40..67024a36f27677f6b9046566c692aca303176e0e 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -131,7 +131,6 @@ "Ubuntu", "Ultra Auth", "Unicorn", - "unicorn-worker-killer", "URL", "WebdriverIO", "YAML", diff --git a/Gemfile b/Gemfile index 724c7143119ff1a344da329780367f3523401c55..089c119f15d77915ac81ee44c0b95d689b912b84 100644 --- a/Gemfile +++ b/Gemfile @@ -185,7 +185,6 @@ gem 'rack-timeout', '~> 0.5.1', require: 'rack/timeout/base' group :unicorn do gem 'unicorn', '~> 5.5' - gem 'unicorn-worker-killer', '~> 0.4.4' end group :puma do diff --git a/Gemfile.lock b/Gemfile.lock index 2d13bfb68c12378278034c23e34bd8d91edea62d..27311e011957f0872c63153611d8f8ab5f3d8012 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1318,9 +1318,6 @@ GEM unicorn (5.5.5) kgio (~> 2.6) raindrops (~> 0.7) - unicorn-worker-killer (0.4.4) - get_process_mem (~> 0) - unicorn (>= 4, < 6) uniform_notifier (1.13.0) unleash (0.1.5) murmurhash3 (~> 0.1.6) @@ -1653,7 +1650,6 @@ DEPENDENCIES u2f (~> 0.2.1) unf (~> 0.1.4) unicorn (~> 5.5) - unicorn-worker-killer (~> 0.4.4) unleash (~> 0.1.5) valid_email (~> 0.1) validates_hostname (~> 1.0.11) diff --git a/config.ru b/config.ru index 0c50b3fdf6fca5fef9efe59c1eec354688b1aeab..ed76239ef2e8d993c86f5f6e083ef7d03621338a 100644 --- a/config.ru +++ b/config.ru @@ -2,25 +2,10 @@ # This file is used by Rack-based servers to start the application. -if defined?(Unicorn) - require 'unicorn' - - if ENV['RAILS_ENV'] == 'production' || ENV['RAILS_ENV'] == 'staging' - # Unicorn self-process killer - require 'unicorn/worker_killer' - - min = (ENV['GITLAB_UNICORN_MEMORY_MIN'] || 400 * 1 << 20).to_i - max = (ENV['GITLAB_UNICORN_MEMORY_MAX'] || 650 * 1 << 20).to_i - - # Max memory size (RSS) per worker - use Unicorn::WorkerKiller::Oom, min, max - end -end - require ::File.expand_path('../config/environment', __FILE__) def master_process? - Prometheus::PidProvider.worker_id.in? %w(unicorn_master puma_master) + Prometheus::PidProvider.worker_id == 'puma_master' end warmup do |app| diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml index 774254d2ff1210fce2364af1810641a39c20308c..d76dd8fe25b82236ced518ca0e852cef4c55c762 100644 --- a/config/dependency_decisions.yml +++ b/config/dependency_decisions.yml @@ -130,13 +130,6 @@ :why: http://unicorn.bogomips.org/LICENSE.html :versions: [] :when: 2016-05-02 05:45:28.817510000 Z -- - :license - - unicorn-worker-killer - - ruby - - :who: Connor Shea - :why: https://github.com/kzk/unicorn-worker-killer/blob/master/LICENSE - :versions: [] - :when: 2016-05-02 05:45:38.323867000 Z - - :license - unf - BSD diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index a168584e75497562e704139d47b28a06cb27e6eb..f98cddb52c90a63c41fa22a7f572a079a4d5224e 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -32,8 +32,6 @@ You can use the following environment variables to override certain values: | `GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`). | | `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation. | | `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners. | -| `GITLAB_UNICORN_MEMORY_MAX` | integer | The maximum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | -| `GITLAB_UNICORN_MEMORY_MIN` | integer | The minimum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | | `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | | `UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`). | diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 289e2cb5362dfe7ed04dc7f1a0aad2f5f09de988..576be33a9be9992b4f383de3e3f36e134bbf6403 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -611,41 +611,6 @@ This file lives in `/var/log/gitlab/puma/puma_stderr.log` for Omnibus GitLab packages, or in `/home/git/gitlab/log/puma_stderr.log` for installations from source. -## Unicorn Logs - -Starting with GitLab 13.0, Puma is the default web server used in GitLab -all-in-one package based installations, and GitLab Helm chart deployments. - -### `unicorn_stdout.log` - -This file lives in `/var/log/gitlab/unicorn/unicorn_stdout.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stdout.log` for -for installations from source. - -### `unicorn_stderr.log` - -This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` for -for installations from source. - -These logs contain all information about the state of Unicorn processes at any given time. - -```plaintext -I, [2015-02-13T06:14:46.680381 #9047] INFO -- : Refreshing Gem list -I, [2015-02-13T06:14:56.931002 #9047] INFO -- : listening on addr=127.0.0.1:8080 fd=12 -I, [2015-02-13T06:14:56.931381 #9047] INFO -- : listening on addr=/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket fd=13 -I, [2015-02-13T06:14:56.936638 #9047] INFO -- : master process ready -I, [2015-02-13T06:14:56.946504 #9092] INFO -- : worker=0 spawned pid=9092 -I, [2015-02-13T06:14:56.946943 #9092] INFO -- : worker=0 ready -I, [2015-02-13T06:14:56.947892 #9094] INFO -- : worker=1 spawned pid=9094 -I, [2015-02-13T06:14:56.948181 #9094] INFO -- : worker=1 ready -W, [2015-02-13T07:16:01.312916 #9094] WARN -- : #: worker (pid: 9094) exceeds memory limit (320626688 bytes > 247066940 bytes) -W, [2015-02-13T07:16:01.313000 #9094] WARN -- : Unicorn::WorkerKiller send SIGQUIT (pid: 9094) alive: 3621 sec (trial 1) -I, [2015-02-13T07:16:01.530733 #9047] INFO -- : reaped # worker=1 -I, [2015-02-13T07:16:01.534501 #13379] INFO -- : worker=1 spawned pid=13379 -I, [2015-02-13T07:16:01.534848 #13379] INFO -- : worker=1 ready -``` - ## `repocheck.log` This file lives in `/var/log/gitlab/gitlab-rails/repocheck.log` for diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 708861d852956e4f55c6fe0c13ff527e9be9ab86..268c9281d8713ec9e806bc8bbea307a627a577e7 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -22,7 +22,6 @@ Keep your GitLab instance up and running smoothly. to restart Sidekiq. - [Multiple Sidekiq processes](extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that need to be processed. **(FREE SELF)** - [Puma](puma.md): Understand Puma and puma-worker-killer. -- [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer. - Speed up SSH operations by [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or by [doing away with user SSH keys stored on GitLab entirely in favor diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md deleted file mode 100644 index 03995ee05ba4acaa2bfbfa9cfdc1e3034c99e98d..0000000000000000000000000000000000000000 --- a/doc/administration/operations/unicorn.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -stage: Enablement -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Understanding Unicorn and unicorn-worker-killer - -NOTE: -Starting with GitLab 13.0, Puma is the default web server used in GitLab -all-in-one package based installations as well as GitLab Helm chart deployments. - -## Unicorn - -GitLab uses [Unicorn](https://yhbt.net/unicorn/), a pre-forking Ruby web -server, to handle web requests (web browsers and Git HTTP clients). Unicorn is -a daemon written in Ruby and C that can load and run a Ruby on Rails -application; in our case the Rails application is GitLab Community Edition or -GitLab Enterprise Edition. - -Unicorn has a multi-process architecture to make better use of available CPU -cores (processes can run on different cores) and to have stronger fault -tolerance (most failures stay isolated in only one process and cannot take down -GitLab entirely). On startup, the Unicorn 'master' process loads a clean Ruby -environment with the GitLab application code, and then spawns 'workers' which -inherit this clean initial environment. The 'master' never handles any -requests, that is left to the workers. The operating system network stack -queues incoming requests and distributes them among the workers. - -In a perfect world, the master would spawn its pool of workers once, and then -the workers handle incoming web requests one after another until the end of -time. In reality, worker processes can crash or time out: if the master notices -that a worker takes too long to handle a request it will terminate the worker -process with SIGKILL ('kill -9'). No matter how the worker process ended, the -master process will replace it with a new 'clean' process again. Unicorn is -designed to be able to replace 'crashed' workers without dropping user -requests. - -This is what a Unicorn worker timeout looks like in `unicorn_stderr.log`. The -master process has PID 56227 below. - -```plaintext -[2015-06-05T10:58:08.660325 #56227] ERROR -- : worker=10 PID:53009 timeout (61s > 60s), killing -[2015-06-05T10:58:08.699360 #56227] ERROR -- : reaped # worker=10 -[2015-06-05T10:58:08.708141 #62538] INFO -- : worker=10 spawned pid=62538 -[2015-06-05T10:58:08.708824 #62538] INFO -- : worker=10 ready -``` - -### Tunable options - -The main tunable options for Unicorn are the number of worker processes and the -request timeout after which the Unicorn master terminates a worker process. -See the [Omnibus GitLab Unicorn settings -documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html) -if you want to adjust these settings. - -## unicorn-worker-killer - -GitLab has memory leaks. These memory leaks manifest themselves in long-running -processes, such as Unicorn workers. (The Unicorn master process is not known to -leak memory, probably because it does not handle user requests.) - -To make these memory leaks manageable, GitLab comes with the -[unicorn-worker-killer gem](https://github.com/kzk/unicorn-worker-killer). This -gem [monkey-patches](https://en.wikipedia.org/wiki/Monkey_patch) the Unicorn -workers to do a memory self-check after every 16 requests. If the memory of the -Unicorn worker exceeds a pre-set limit then the worker process exits. The -Unicorn master then automatically replaces the worker process. - -This is a robust way to handle memory leaks: Unicorn is designed to handle -workers that 'crash' so no user requests will be dropped. The -unicorn-worker-killer gem is designed to only terminate a worker process _in -between requests_, so no user requests are affected. You can set the minimum and -maximum memory threshold (in bytes) for the Unicorn worker killer by -setting the following values `/etc/gitlab/gitlab.rb`: - -- For GitLab **12.7** and newer: - - ```ruby - unicorn['worker_memory_limit_min'] = "1024 * 1 << 20" - unicorn['worker_memory_limit_max'] = "1280 * 1 << 20" - ``` - -- For GitLab **12.6** and older: - - ```ruby - unicorn['worker_memory_limit_min'] = "400 * 1 << 20" - unicorn['worker_memory_limit_max'] = "650 * 1 << 20" - ``` - -Otherwise, you can set the `GITLAB_UNICORN_MEMORY_MIN` and `GITLAB_UNICORN_MEMORY_MAX` -[environment variables](../environment_variables.md). - -This is what a Unicorn worker memory restart looks like in unicorn_stderr.log. -You see that worker 4 (PID 125918) is inspecting itself and decides to exit. -The threshold memory value was 254802235 bytes, about 250MB. With GitLab this -threshold is a random value between 200 and 250 MB. The master process (PID -117565) then reaps the worker process and spawns a new 'worker 4' with PID -127549. - -```plaintext -[2015-06-05T12:07:41.828374 #125918] WARN -- : #: worker (pid: 125918) exceeds memory limit (256413696 bytes > 254802235 bytes) -[2015-06-05T12:07:41.828472 #125918] WARN -- : Unicorn::WorkerKiller send SIGQUIT (pid: 125918) alive: 23 sec (trial 1) -[2015-06-05T12:07:42.025916 #117565] INFO -- : reaped # worker=4 -[2015-06-05T12:07:42.034527 #127549] INFO -- : worker=4 spawned pid=127549 -[2015-06-05T12:07:42.035217 #127549] INFO -- : worker=4 ready -``` - -One other thing that stands out in the log snippet above, taken from -GitLab.com, is that 'worker 4' was serving requests for only 23 seconds. This -is a normal value for our current GitLab.com setup and traffic. - -The high frequency of Unicorn memory restarts on some GitLab sites can be a -source of confusion for administrators. Usually they are a [red -herring](https://en.wikipedia.org/wiki/Red_herring). diff --git a/doc/development/architecture.md b/doc/development/architecture.md index fdcaa91a639df876f89feb52ca48de307b2fecce..d8d452240ff38250fd6b4e091a8777695deeb41d 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -710,23 +710,6 @@ disabled by default. [Puma](https://puma.io/) is a Ruby application server that is used to run the core Rails Application that provides the user facing features in GitLab. Often this displays in process output as `bundle` or `config.ru` depending on the GitLab version. -#### Unicorn - -Starting with GitLab 13.0, Puma is the default web server and Unicorn has been -disabled by default. - -- [Project page](https://gitlab.com/gitlab-org/gitlab/blob/master/README.md) -- Configuration: - - [Omnibus](https://docs.gitlab.com/omnibus/settings/unicorn.html) - - [Charts](https://docs.gitlab.com/charts/charts/gitlab/webservice/) - - [Source](../install/installation.md#configure-it) - - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) -- Layer: Core Service (Processor) -- Process: `unicorn` -- GitLab.com: [Unicorn](../user/gitlab_com/index.md#unicorn) - -[Unicorn](https://yhbt.net/unicorn/) is a Ruby application server that is used to run the core Rails Application that provides the user facing features in GitLab. Often this displays in process output as `bundle` or `config.ru` depending on the GitLab version. - #### LDAP Authentication - Configuration: diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index cdb4ca52c9c08999a1c48852cee1cdce2788c088..f05e5f00bc2d79ffa4e67bfd79613a83b082a0ea 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -508,20 +508,6 @@ of proposed changes can be found at GitLab.com uses the default of 60 seconds for [Puma request timeouts](https://docs.gitlab.com/omnibus/settings/puma.html#worker-timeout). -## Unicorn - -GitLab.com adjusts the memory limits for the [unicorn-worker-killer](https://rubygems.org/gems/unicorn-worker-killer) gem. - -Base default: - -- `memory_limit_min` = 750MiB -- `memory_limit_max` = 1024MiB - -Web front-ends: - -- `memory_limit_min` = 1024MiB -- `memory_limit_max` = 1280MiB - ## GitLab.com-specific rate limits NOTE: