From 822995446828837727d7b703ec3dd7dba53a04b8 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 24 May 2017 19:13:05 +0800 Subject: [PATCH 01/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- .gitignore | 4 + .gitlab-ci.yml | 28 +++ app/controllers/health_controller.rb | 4 +- app/models/ci/runner.rb | 4 +- app/models/project.rb | 8 +- config/application.rb | 18 +- config/caching.yml.example | 40 ++++ config/initializers/7_redis.rb | 4 +- config/initializers/session_store.rb | 6 +- config/initializers/sidekiq.rb | 12 +- config/mail_room.yml | 4 +- ...{resque.yml.example => queues.yml.example} | 20 +- config/shared_state.yml.example | 40 ++++ ...er_activities_to_users_last_activity_on.rb | 4 +- .../operations/cleaning_up_redis_sessions.md | 2 +- lib/gitlab/auth/unique_ips_limiter.rb | 4 +- .../cache/ci/project_pipeline_status.rb | 24 +- lib/gitlab/caching.rb | 22 ++ lib/gitlab/chat_name_token.rb | 14 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/etag_caching/store.rb | 12 +- lib/gitlab/exclusive_lease.rb | 16 +- lib/gitlab/health_checks/caching_check.rb | 25 +++ .../{redis_check.rb => queues_check.rb} | 6 +- .../health_checks/shared_state_check.rb | 25 +++ lib/gitlab/lfs_token.rb | 8 +- lib/gitlab/mail_room.rb | 11 +- lib/gitlab/queues.rb | 23 ++ lib/gitlab/{redis.rb => redis_wrapper.rb} | 42 ++-- lib/gitlab/shared_state.rb | 22 ++ lib/gitlab/user_activities.rb | 12 +- lib/gitlab/workhorse.rb | 8 +- lib/tasks/cache.rake | 25 ++- locale/gitlab.pot | 4 +- scripts/prepare_build.sh | 28 +++ spec/config/mail_room_spec.rb | 22 +- spec/controllers/health_controller_spec.rb | 30 ++- spec/controllers/sessions_controller_spec.rb | 2 +- spec/features/dashboard/projects_spec.rb | 2 +- spec/features/groups/members/sorting_spec.rb | 4 +- spec/features/login_spec.rb | 4 +- .../features/projects/members/sorting_spec.rb | 4 +- .../config/caching_config_with_env.yml | 2 + ...t_host.yml => caching_new_format_host.yml} | 12 +- .../config/caching_new_format_socket.yml | 6 + .../config/caching_old_format_host.yml | 5 + .../config/caching_old_format_socket.yml | 3 + .../config/queues_config_with_env.yml | 2 + .../config/queues_new_format_host.yml | 29 +++ .../config/queues_new_format_socket.yml | 6 + .../config/queues_old_format_host.yml | 5 + .../config/queues_old_format_socket.yml | 3 + .../fixtures/config/redis_config_with_env.yml | 2 - .../config/redis_new_format_socket.yml | 6 - .../fixtures/config/redis_old_format_host.yml | 5 - .../config/redis_old_format_socket.yml | 3 - .../config/shared_state_config_with_env.yml | 2 + .../config/shared_state_new_format_host.yml | 29 +++ .../config/shared_state_new_format_socket.yml | 6 + .../config/shared_state_old_format_host.yml | 5 + .../config/shared_state_old_format_socket.yml | 3 + spec/helpers/projects_helper_spec.rb | 2 +- .../gitlab/auth/unique_ips_limiter_spec.rb | 2 +- .../cache/ci/project_pipeline_status_spec.rb | 58 ++--- spec/lib/gitlab/caching_spec.rb | 211 ++++++++++++++++++ spec/lib/gitlab/current_settings_spec.rb | 6 +- spec/lib/gitlab/exclusive_lease_spec.rb | 2 +- .../health_checks/caching_check_spec.rb | 6 + .../gitlab/health_checks/queues_check_spec.rb | 6 + .../gitlab/health_checks/redis_check_spec.rb | 6 - .../health_checks/shared_state_check_spec.rb | 6 + .../health_checks/simple_check_shared.rb | 4 +- .../gitlab/{redis_spec.rb => queues_spec.rb} | 52 ++--- spec/lib/gitlab/shared_state_spec.rb | 211 ++++++++++++++++++ spec/lib/gitlab/sidekiq_status_spec.rb | 12 +- spec/lib/gitlab/user_activities_spec.rb | 54 ++--- spec/lib/gitlab/workhorse_spec.rb | 4 +- ...migrate_process_commit_worker_jobs_spec.rb | 4 +- ...tivities_to_users_last_activity_on_spec.rb | 6 +- spec/models/ci/runner_spec.rb | 18 +- spec/models/user_spec.rb | 2 +- spec/requests/api/internal_spec.rb | 2 +- spec/requests/api/users_spec.rb | 2 +- spec/requests/git_http_spec.rb | 2 +- spec/requests/openid_connect_spec.rb | 2 +- spec/services/event_create_service_spec.rb | 2 +- spec/services/git_push_service_spec.rb | 10 +- .../services/projects/destroy_service_spec.rb | 4 +- spec/services/users/activity_service_spec.rb | 2 +- ...efresh_authorized_projects_service_spec.rb | 2 +- spec/spec_helper.rb | 13 +- .../unique_ip_check_shared_examples.rb | 4 +- ...hedule_update_user_activity_worker_spec.rb | 2 +- .../update_user_activity_worker_spec.rb | 4 +- 94 files changed, 1139 insertions(+), 317 deletions(-) create mode 100644 config/caching.yml.example rename config/{resque.yml.example => queues.yml.example} (59%) create mode 100644 config/shared_state.yml.example create mode 100644 lib/gitlab/caching.rb create mode 100644 lib/gitlab/health_checks/caching_check.rb rename lib/gitlab/health_checks/{redis_check.rb => queues_check.rb} (79%) create mode 100644 lib/gitlab/health_checks/shared_state_check.rb create mode 100644 lib/gitlab/queues.rb rename lib/gitlab/{redis.rb => redis_wrapper.rb} (69%) create mode 100644 lib/gitlab/shared_state.rb create mode 100644 spec/fixtures/config/caching_config_with_env.yml rename spec/fixtures/config/{redis_new_format_host.yml => caching_new_format_host.yml} (62%) create mode 100644 spec/fixtures/config/caching_new_format_socket.yml create mode 100644 spec/fixtures/config/caching_old_format_host.yml create mode 100644 spec/fixtures/config/caching_old_format_socket.yml create mode 100644 spec/fixtures/config/queues_config_with_env.yml create mode 100644 spec/fixtures/config/queues_new_format_host.yml create mode 100644 spec/fixtures/config/queues_new_format_socket.yml create mode 100644 spec/fixtures/config/queues_old_format_host.yml create mode 100644 spec/fixtures/config/queues_old_format_socket.yml delete mode 100644 spec/fixtures/config/redis_config_with_env.yml delete mode 100644 spec/fixtures/config/redis_new_format_socket.yml delete mode 100644 spec/fixtures/config/redis_old_format_host.yml delete mode 100644 spec/fixtures/config/redis_old_format_socket.yml create mode 100644 spec/fixtures/config/shared_state_config_with_env.yml create mode 100644 spec/fixtures/config/shared_state_new_format_host.yml create mode 100644 spec/fixtures/config/shared_state_new_format_socket.yml create mode 100644 spec/fixtures/config/shared_state_old_format_host.yml create mode 100644 spec/fixtures/config/shared_state_old_format_socket.yml create mode 100644 spec/lib/gitlab/caching_spec.rb create mode 100644 spec/lib/gitlab/health_checks/caching_check_spec.rb create mode 100644 spec/lib/gitlab/health_checks/queues_check_spec.rb delete mode 100644 spec/lib/gitlab/health_checks/redis_check_spec.rb create mode 100644 spec/lib/gitlab/health_checks/shared_state_check_spec.rb rename spec/lib/gitlab/{redis_spec.rb => queues_spec.rb} (74%) create mode 100644 spec/lib/gitlab/shared_state_spec.rb diff --git a/.gitignore b/.gitignore index 89da29fd7904..9f0e0298fb58 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ eslint-report.html /config/initializers/smtp_settings.rb /config/initializers/relative_url.rb /config/resque.yml +/config/caching.yml +/config/queues.yml +/config/shared_state.yml /config/unicorn.rb /config/secrets.yml /config/sidekiq.yml @@ -58,4 +61,5 @@ eslint-report.html /shared/* /.gitlab_workhorse_secret /webpack-report/ +/locale/gitlab.pot /locale/**/LC_MESSAGES diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 638553d7bf75..31d7ba4ae5b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,11 +54,39 @@ stages: services: - postgres:9.2 - redis:alpine +# +# notes for when CI supports service configuration on shared image +# - caching: +# image: redis:alpine +# ports: +# - "6380:6379" +# - queues: +# image: redis:alpine +# ports: +# - "6381:6379" +# - shared_state: +# image: redis:alpine +# ports: +# - "6382:6379" .use-mysql: &use-mysql services: - mysql:latest - redis:alpine +# +# notes for when CI supports service configuration on shared image +# - caching: +# image: redis:alpine +# ports: +# - "6380:6379" +# - queues: +# image: redis:alpine +# ports: +# - "6381:6379" +# - shared_state: +# image: redis:alpine +# ports: +# - "6382:6379" .only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql only: diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 125746d04264..ae7fe80814c3 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,7 +4,9 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCheck, + Gitlab::HealthChecks::CachingCheck, + Gitlab::HealthChecks::QueuesCheck, + Gitlab::HealthChecks::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 487ba61bc9c7..3b660e65a37f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,8 +145,8 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::Redis.with do |redis| - redis.del(runner_queue_key) + Gitlab::Queues.with do |queues| + queues.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index 65745fd6d370..b3d92be10cf1 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1282,15 +1282,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i } + Gitlab::SharedState.with { |shared_state| shared_state.get(pushes_since_gc_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) } + Gitlab::SharedState.with { |shared_state| shared_state.incr(pushes_since_gc_shared_state_key) } end def reset_pushes_since_gc - Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } + Gitlab::SharedState.with { |shared_state| shared_state.del(pushes_since_gc_shared_state_key) } end def route_map_for(commit_sha) @@ -1353,7 +1353,7 @@ def cross_project_reference?(from) from && self != from end - def pushes_since_gc_redis_key + def pushes_since_gc_shared_state_key "projects/#{id}/pushes_since_gc" end diff --git a/config/application.rb b/config/application.rb index 95ba67749167..390fdab118ed 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,7 +6,9 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis') + require_dependency Rails.root.join('lib/gitlab/caching') + require_dependency Rails.root.join('lib/gitlab/queues') + require_dependency Rails.root.join('lib/gitlab/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -137,15 +139,15 @@ class Application < Rails::Application end end - # Use Redis caching across all environments - redis_config_hash = Gitlab::Redis.params - redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE - redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever + # Use caching across all environments + caching_config_hash = Gitlab::Caching.params + caching_config_hash[:namespace] = Gitlab::Caching::CACHE_NAMESPACE + caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context - redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 - redis_config_hash[:pool_timeout] = 1 + caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 + caching_config_hash[:pool_timeout] = 1 end - config.cache_store = :redis_store, redis_config_hash + config.cache_store = :redis_store, caching_config_hash config.active_record.raise_in_transactional_callbacks = true diff --git a/config/caching.yml.example b/config/caching.yml.example new file mode 100644 index 000000000000..5fe9b8f1adb1 --- /dev/null +++ b/config/caching.yml.example @@ -0,0 +1,40 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/10 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6380 + # sentinels: + # - + # host: localhost + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/10 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6380 +production: + # Redis (single instance) + url: unix:/var/run/redis/caching.redis.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6380 + # sentinels: + # - + # host: slave1 + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index ae2ca258df18..9dfcae2b07f0 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,3 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::Redis.with { nil } +Gitlab::Caching.with { nil } +Gitlab::Queues.with { nil } +Gitlab::SharedState.with { nil } diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 70be2617cabf..0181003160f7 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,12 +13,12 @@ if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - redis_config = Gitlab::Redis.params - redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE + sessions_config = Gitlab::SharedState.params + sessions_config[:namespace] = Gitlab::SharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. - servers: redis_config, + servers: sessions_config, key: '_gitlab_session', secure: Gitlab.config.gitlab.https, httponly: true, diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index ecd739564883..4d3b2074aece 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,12 +1,12 @@ -# Custom Redis configuration -redis_config_hash = Gitlab::Redis.params -redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE +# Custom Queues configuration +queues_config_hash = Gitlab::Queues.params +queues_config_hash[:namespace] = Gitlab::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } Sidekiq.configure_server do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] @@ -54,7 +54,7 @@ end Sidekiq.configure_client do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.client_middleware do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware @@ -74,5 +74,5 @@ end end end -rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue Gitlab::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/config/mail_room.yml b/config/mail_room.yml index 88d93d4bc6b0..24f34109a03b 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::Queues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::Queues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/config/resque.yml.example b/config/queues.yml.example similarity index 59% rename from config/resque.yml.example rename to config/queues.yml.example index 0c19d8bc1d36..25284393d7e4 100644 --- a/config/resque.yml.example +++ b/config/queues.yml.example @@ -2,19 +2,25 @@ # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # development: - url: redis://localhost:6379 + url: redis://localhost:6379/11 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6381 # sentinels: # - # host: localhost - # port: 26380 # point to sentinel, not to redis port + # port: 26381 # point to sentinel, not to redis port # - # host: slave2 # port: 26381 # point to sentinel, not to redis port test: - url: redis://localhost:6379 + url: redis://localhost:6379/11 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6381 production: # Redis (single instance) - url: unix:/var/run/redis/redis.sock + url: unix:/var/run/redis/queues.redis.sock ## # Redis + Sentinel (for HA) # @@ -24,11 +30,11 @@ production: # You must specify a list of a few sentinels that will handle client connection # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html ## - # url: redis://master:6379 + # url: redis://master:6381 # sentinels: # - # host: slave1 - # port: 26379 # point to sentinel, not to redis port + # port: 26381 # point to sentinel, not to redis port # - # host: slave2 - # port: 26379 # point to sentinel, not to redis port + # port: 26381 # point to sentinel, not to redis port diff --git a/config/shared_state.yml.example b/config/shared_state.yml.example new file mode 100644 index 000000000000..f1a6868aac4b --- /dev/null +++ b/config/shared_state.yml.example @@ -0,0 +1,40 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/12 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6382 + # sentinels: + # - + # host: localhost + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/12 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6382 +production: + # Redis (single instance) + url: unix:/var/run/redis/shared_state.redis.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6382 + # sentinels: + # - + # host: slave1 + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb index 9ad36482c8a3..7152857a7247 100644 --- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -56,7 +56,7 @@ def users_table end def activities(from, to, page: 1) - Gitlab::Redis.with do |redis| + Gitlab::SharedState.with do |redis| redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, with_scores: true, limit: limit(page)) @@ -64,7 +64,7 @@ def activities(from, to, page: 1) end def activities_count(from, to) - Gitlab::Redis.with do |redis| + Gitlab::SharedState.with do |redis| redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) end end diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 93521e976d51..253391604727 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -22,7 +22,7 @@ rcli() { # This example works for Omnibus installations of GitLab 7.3 or newer. For an # installation from source you will have to change the socket path and the # path to redis-cli. - sudo /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.socket "$@" + sudo /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/shared_state.redis.socket "$@" } # test the new shell function; the response should be PONG diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150e..2cf3198324aa 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,9 +27,9 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::SharedState.with do |shared_state| unique_ips_count = nil - redis.multi do |r| + shared_state.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) unique_ips_count = r.zcard(key) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 4fc9a075edc8..dee1a004614e 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,12 +23,12 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| - redis.multi do + result = Gitlab::Caching.with do |caching| + caching.multi do projects.each do |project| cache_key = cache_key_for_project(project) - redis.exists(cache_key) - redis.hmget(cache_key, :sha, :status, :ref) + caching.exists(cache_key) + caching.hmget(cache_key, :sha, :status, :ref) end end end @@ -100,28 +100,28 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::Redis.with do |redis| - self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + Gitlab::Caching.with do |caching| + self.sha, self.status, self.ref = caching.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::Caching.with do |caching| + caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| - redis.del(cache_key) + Gitlab::Caching.with do |caching| + caching.del(cache_key) end end def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| - redis.exists(cache_key) + Gitlab::Caching.with do |caching| + caching.exists(cache_key) end end diff --git a/lib/gitlab/caching.rb b/lib/gitlab/caching.rb new file mode 100644 index 000000000000..84c8864d6cd0 --- /dev/null +++ b/lib/gitlab/caching.rb @@ -0,0 +1,22 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'redis_wrapper' + +module Gitlab + class Caching < ::Gitlab::RedisWrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_CACHING_CONFIG_FILE'] || File.expand_path('../../config/caching.yml', __dir__) + end + + def default_url + DEFAULT_CACHING_URL + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d7..b298fae8c275 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ def initialize(token = new_token) end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::SharedState.with do |shared_state| + data = shared_state.get(shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::SharedState.with do |shared_state| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + shared_state.set(shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::SharedState.with do |shared_state| + shared_state.del(shared_state_key) end end @@ -38,7 +38,7 @@ def new_token Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fed..ea11a5d1cea6 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Gitlab::Caching::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 0039fc01c8f0..939c16e68c21 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::SharedState.with { |shared_state| shared_state.get(shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::SharedState.with do |shared_state| + shared_state.set(shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,8 +24,8 @@ def generate_etag SecureRandom.hex end - def redis_key(key) - "#{REDIS_NAMESPACE}#{key}" + def shared_state_key(key) + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 62ddd45785d9..efbb13221db1 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,17 +18,17 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::SharedState.with do |shared_state| + shared_state.eval(LUA_CANCEL_SCRIPT, keys: [shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @shared_state_key = self.class.shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -37,15 +37,15 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::SharedState.with do |shared_state| + shared_state.set(@shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::SharedState.with do |shared_state| + shared_state.exists(@shared_state_key) end end end diff --git a/lib/gitlab/health_checks/caching_check.rb b/lib/gitlab/health_checks/caching_check.rb new file mode 100644 index 000000000000..155457403dd6 --- /dev/null +++ b/lib/gitlab/health_checks/caching_check.rb @@ -0,0 +1,25 @@ +module Gitlab + module HealthChecks + class CachingCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'caching_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Caching.with(&:ping) + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/queues_check.rb similarity index 79% rename from lib/gitlab/health_checks/redis_check.rb rename to lib/gitlab/health_checks/queues_check.rb index 57bbe5b3ad03..651949532415 100644 --- a/lib/gitlab/health_checks/redis_check.rb +++ b/lib/gitlab/health_checks/queues_check.rb @@ -1,13 +1,13 @@ module Gitlab module HealthChecks - class RedisCheck + class QueuesCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'redis_ping' + 'queues_ping' end def is_successful?(result) @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) + Gitlab::Queues.with(&:ping) end end end diff --git a/lib/gitlab/health_checks/shared_state_check.rb b/lib/gitlab/health_checks/shared_state_check.rb new file mode 100644 index 000000000000..194dabc8b58e --- /dev/null +++ b/lib/gitlab/health_checks/shared_state_check.rb @@ -0,0 +1,25 @@ +module Gitlab + module HealthChecks + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::SharedState.with(&:ping) + end + end + end + end + end +end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a7..a1cf9daf69bb 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ def initialize(actor) end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::SharedState.with do |shared_state| + token = shared_state.get(shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + shared_state.set(shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ def actor_name private - def redis_key + def shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8b..7fd2e0da7bfc 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'queues' unless defined?(Gitlab::Queues) module Gitlab module MailRoom @@ -34,11 +34,12 @@ def fetch_config config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_queues = Gitlab::Queues.new(rails_env) + config[:redis_url] = gitlab_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + # @@ TODO this is too-much-information on topology in the client of an object + if gitlab_queues.sentinels? + config[:sentinels] = gitlab_queues.sentinels end end diff --git a/lib/gitlab/queues.rb b/lib/gitlab/queues.rb new file mode 100644 index 000000000000..661f886fecc2 --- /dev/null +++ b/lib/gitlab/queues.rb @@ -0,0 +1,23 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'redis_wrapper' + +module Gitlab + class Queues < ::Gitlab::RedisWrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_QUEUES_CONFIG_FILE'] || File.expand_path('../../config/queues.yml', __dir__) + end + + def default_url + DEFAULT_QUEUES_URL + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis_wrapper.rb similarity index 69% rename from lib/gitlab/redis.rb rename to lib/gitlab/redis_wrapper.rb index bc5370de32aa..88976938898e 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis_wrapper.rb @@ -1,15 +1,15 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes # This file should not have any direct dependency on Rails environment # please require all dependencies below: +require 'rails' +require 'redis' require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + class RedisWrapper + BaseError = ::Redis::BaseError.freeze + CannotConnectError = ::Redis::CannotConnectError.freeze class << self delegate :params, :url, to: :new @@ -20,30 +20,26 @@ def with end def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 if Sidekiq.server? # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 + size += Sidekiq.options[:concurrency] end + size end + end - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end + def _raw_config + return @_raw_config if defined?(@_raw_config) - @_raw_config + begin + @_raw_config = ERB.new(File.read(config_file)).result.freeze + rescue Errno::ENOENT + @_raw_config = false end - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end + @_raw_config end def initialize(rails_env = nil) @@ -91,7 +87,7 @@ def raw_config_hash if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: DEFAULT_REDIS_URL } + { url: default_url } end end diff --git a/lib/gitlab/shared_state.rb b/lib/gitlab/shared_state.rb new file mode 100644 index 000000000000..25381dc36611 --- /dev/null +++ b/lib/gitlab/shared_state.rb @@ -0,0 +1,22 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'redis_wrapper' + +module Gitlab + class SharedState < ::Gitlab::RedisWrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../config/shared_state.yml', __dir__) + end + + def default_url + DEFAULT_SHARED_STATE_URL + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fdedf..2366071ec03f 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,14 +6,14 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| - redis.hset(KEY, key, time.to_i) + Gitlab::SharedState.with do |shared_state| + shared_state.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| - redis.hdel(KEY, keys) + Gitlab::SharedState.with do |shared_state| + shared_state.hdel(KEY, keys) end end @@ -21,8 +21,8 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| - redis.hscan(KEY, cursor, count: BATCH_SIZE) + Gitlab::SharedState.with do |shared_state| + shared_state.hscan(KEY, cursor, count: BATCH_SIZE) end Hash[pairs].each { |pair| yield pair } diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index fe37e4da94f2..5e1f15fa7b7d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,13 +174,13 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| - result = redis.set(key, value, ex: expire, nx: !overwrite) + Gitlab::Queues.with do |queues| + result = queues.set(key, value, ex: expire, nx: !overwrite) if result - redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") value else - redis.get(key) + queues.get(key) end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d66..e31011ca8ce1 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,23 @@ namespace :cache do namespace :clear do - REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan + # @@ TODO - push implementation into Gitlab::Caching + KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task redis: :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP + desc "GitLab | Clear key/value cache" + task gitlab_kv_cache: :environment do + Gitlab::Caching.with do |kv_cache| + cursor = KV_CACHE_SCAN_START_STOP loop do - cursor, keys = redis.scan( + cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: REDIS_CLEAR_BATCH_SIZE + match: "#{Gitlab::Caching::CACHE_NAMESPACE}*", + count: KV_CACHE_CLEAR_BATCH_SIZE ) - redis.del(*keys) if keys.any? + kv_cache.del(*keys) if keys.any? - break if cursor == REDIS_SCAN_START_STOP + break if cursor == KV_CACHE_SCAN_START_STOP end end end @@ -24,5 +25,5 @@ namespace :cache do task all: [:redis] end - task clear: 'cache:clear:redis' + task clear: 'cache:clear:gitlab_kv_cache' end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3967d40ea9e5..a64d6b36ebf4 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8,8 +8,8 @@ msgid "" msgstr "" "Project-Id-Version: gitlab 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-05-04 19:24-0500\n" -"PO-Revision-Date: 2017-05-04 19:24-0500\n" +"POT-Creation-Date: 2017-05-24 19:07+0800\n" +"PO-Revision-Date: 2017-05-24 19:07+0800\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 03de59f27adc..db331e112936 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -37,6 +37,34 @@ else # Assume it's mysql sed -i 's/# host:.*/host: mysql/g' config/database.yml fi +cp config/caching.yml.example config/caching.yml +sed -i 's/localhost/redis/g' config/caching.yml +# +# notes for when CI supports service configuration on shared image +#sed -i 's/localhost/caching/g' config/caching.yml + +cp config/queues.yml.example config/queues.yml +sed -i 's/localhost/redis/g' config/queues.yml +# +# notes for when CI supports service configuration on shared image +#sed -i 's/localhost/queues/g' config/queues.yml + +cp config/shared_state.yml.example config/shared_state.yml +sed -i 's/localhost/redis/g' config/shared_state.yml +# +# notes for when CI supports service configuration on shared image +#sed -i 's/localhost/shared_state/g' config/shared_state.yml + +cp config/gitlab.yml.example config/gitlab.yml + +if [ "$USE_BUNDLE_INSTALL" != "false" ]; then + bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check +fi + +# Only install knapsack after bundle install! Otherwise oddly some native +# gems could not be found under some circumstance. No idea why, hours wasted. +retry gem install knapsack fog-aws mime-types + if [ "$SETUP_DB" != "false" ]; then bundle exec rake db:drop db:create db:schema:load db:migrate diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index 092048a62599..a30f340f5242 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -5,12 +5,12 @@ let(:mailroom_config_path) { 'config/mail_room.yml' } let(:gitlab_config_path) { 'config/mail_room.yml' } - let(:redis_config_path) { 'config/resque.yml' } + let(:queues_config_path) { 'config/queues.yml' } let(:configuration) do vars = { 'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path), - 'GITLAB_REDIS_CONFIG_FILE' => absolute_path(redis_config_path) + 'GITLAB_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path) } cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result" @@ -21,12 +21,12 @@ end before(:each) do - stub_env('GITLAB_REDIS_CONFIG_FILE', absolute_path(redis_config_path)) - clear_redis_raw_config + stub_env('GITLAB_QUEUES_CONFIG_FILE', absolute_path(queues_config_path)) + clear_queues_raw_config end after(:each) do - clear_redis_raw_config + clear_queues_raw_config end context 'when incoming email is disabled' do @@ -39,9 +39,9 @@ context 'when incoming email is enabled' do let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } - let(:redis_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:queues_config_path) { 'spec/fixtures/config/queues_new_format_host.yml' } - let(:gitlab_redis) { Gitlab::Redis.new(Rails.env) } + let(:gitlab_queues) { Gitlab::Queues.new(Rails.env) } it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) @@ -56,8 +56,8 @@ expect(mailbox[:name]).to eq('inbox') expect(mailbox[:idle_timeout]).to eq(60) - redis_url = gitlab_redis.url - sentinels = gitlab_redis.sentinels + redis_url = gitlab_queues.url + sentinels = gitlab_queues.sentinels expect(mailbox[:delivery_options][:redis_url]).to be_present expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) @@ -73,8 +73,8 @@ end end - def clear_redis_raw_config - Gitlab::Redis.remove_instance_variable(:@_raw_config) + def clear_queues_raw_config + Gitlab::Queues.remove_instance_variable(:@_raw_config) rescue NameError # raised if @_raw_config was not set; ignore end diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index b8b6e0c3a88e..c73639024b41 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,7 +19,9 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') end @@ -42,7 +44,9 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') end end @@ -68,11 +72,25 @@ expect(response.body).to match(/^db_ping_latency [0-9\.]+$/) end - it 'returns Redis ping metrics' do + it 'returns Caching ping metrics' do get :metrics - expect(response.body).to match(/^redis_ping_timeout 0$/) - expect(response.body).to match(/^redis_ping_success 1$/) - expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^caching_ping_timeout 0$/) + expect(response.body).to match(/^caching_ping_success 1$/) + expect(response.body).to match(/^caching_ping_latency [0-9\.]+$/) + end + + it 'returns Queues ping metrics' do + get :metrics + expect(response.body).to match(/^queues_ping_timeout 0$/) + expect(response.body).to match(/^queues_ping_success 1$/) + expect(response.body).to match(/^queues_ping_latency [0-9\.]+$/) + end + + it 'returns SharedState ping metrics' do + get :metrics + expect(response.body).to match(/^shared_state_ping_timeout 0$/) + expect(response.body).to match(/^shared_state_ping_success 1$/) + expect(response.body).to match(/^shared_state_ping_latency [0-9\.]+$/) end it 'returns file system check metrics' do diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index 038132cffe09..e7e59731f662 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -16,7 +16,7 @@ end end - context 'when using valid password', :redis do + context 'when using valid password', :gitlab_shared_state do include UserActivitiesHelpers let(:user) { create(:user) } diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb index f1789fc9d433..18390c2da46a 100644 --- a/spec/features/dashboard/projects_spec.rb +++ b/spec/features/dashboard/projects_spec.rb @@ -14,7 +14,7 @@ expect(page).to have_content('awesome stuff') end - describe "with a pipeline", redis: true do + describe "with a pipeline", gitlab_shared_state: true do let!(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) } before do diff --git a/spec/features/groups/members/sorting_spec.rb b/spec/features/groups/members/sorting_spec.rb index 902d3f789ff1..1f66a8b41c49 100644 --- a/spec/features/groups/members/sorting_spec.rb +++ b/spec/features/groups/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :redis do + scenario 'sorts by recent sign in', :gitlab_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(owner.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :redis do + scenario 'sorts by oldest sign in', :gitlab_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb index c82e8c03343f..8490661d0253 100644 --- a/spec/features/login_spec.rb +++ b/spec/features/login_spec.rb @@ -41,7 +41,7 @@ expect(page).to have_content('Your account has been blocked.') end - it 'does not update Devise trackable attributes', :redis do + it 'does not update Devise trackable attributes', :gitlab_shared_state do user = create(:user, :blocked) expect { login_with(user) }.not_to change { user.reload.sign_in_count } @@ -55,7 +55,7 @@ expect(page).to have_content('Invalid Login or password.') end - it 'does not update Devise trackable attributes', :redis do + it 'does not update Devise trackable attributes', :gitlab_shared_state do expect { login_with(User.ghost) }.not_to change { User.ghost.reload.sign_in_count } end end diff --git a/spec/features/projects/members/sorting_spec.rb b/spec/features/projects/members/sorting_spec.rb index b7ae5f0b9258..acd38598defd 100644 --- a/spec/features/projects/members/sorting_spec.rb +++ b/spec/features/projects/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :redis do + scenario 'sorts by recent sign in', :gitlab_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(master.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :redis do + scenario 'sorts by oldest sign in', :gitlab_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/fixtures/config/caching_config_with_env.yml b/spec/fixtures/config/caching_config_with_env.yml new file mode 100644 index 000000000000..97341956f140 --- /dev/null +++ b/spec/fixtures/config/caching_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_CACHING_URL'] %> diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/caching_new_format_host.yml similarity index 62% rename from spec/fixtures/config/redis_new_format_host.yml rename to spec/fixtures/config/caching_new_format_host.yml index 13772677a45c..902bfa9ca148 100644 --- a/spec/fixtures/config/redis_new_format_host.yml +++ b/spec/fixtures/config/caching_new_format_host.yml @@ -1,29 +1,29 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis development: - url: redis://:mynewpassword@localhost:6379/99 + url: redis://:mynewpassword@localhost:6380/99 sentinels: - host: localhost port: 26380 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26380 # point to sentinel, not to redis port test: - url: redis://:mynewpassword@localhost:6379/99 + url: redis://:mynewpassword@localhost:6380/99 sentinels: - host: localhost port: 26380 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26380 # point to sentinel, not to redis port production: - url: redis://:mynewpassword@localhost:6379/99 + url: redis://:mynewpassword@localhost:6380/99 sentinels: - host: slave1 port: 26380 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26380 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/caching_new_format_socket.yml b/spec/fixtures/config/caching_new_format_socket.yml new file mode 100644 index 000000000000..15b5226ce29b --- /dev/null +++ b/spec/fixtures/config/caching_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/caching.redis.sock +test: + url: unix:/path/to/caching.redis.sock +production: + url: unix:/path/to/caching.redis.sock diff --git a/spec/fixtures/config/caching_old_format_host.yml b/spec/fixtures/config/caching_old_format_host.yml new file mode 100644 index 000000000000..7164e088e56d --- /dev/null +++ b/spec/fixtures/config/caching_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6380/99 +test: redis://:mypassword@localhost:6380/99 +production: redis://:mypassword@localhost:6380/99 diff --git a/spec/fixtures/config/caching_old_format_socket.yml b/spec/fixtures/config/caching_old_format_socket.yml new file mode 100644 index 000000000000..67fac4877e58 --- /dev/null +++ b/spec/fixtures/config/caching_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/caching.redis.sock +test: unix:/path/to/old/caching.redis.sock +production: unix:/path/to/old/caching.redis.sock diff --git a/spec/fixtures/config/queues_config_with_env.yml b/spec/fixtures/config/queues_config_with_env.yml new file mode 100644 index 000000000000..7c5b2b9e69f4 --- /dev/null +++ b/spec/fixtures/config/queues_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_QUEUES_URL'] %> diff --git a/spec/fixtures/config/queues_new_format_host.yml b/spec/fixtures/config/queues_new_format_host.yml new file mode 100644 index 000000000000..d3e99d277df9 --- /dev/null +++ b/spec/fixtures/config/queues_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6381/99 + sentinels: + - + host: localhost + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6381/99 + sentinels: + - + host: localhost + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6381/99 + sentinels: + - + host: slave1 + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/queues_new_format_socket.yml b/spec/fixtures/config/queues_new_format_socket.yml new file mode 100644 index 000000000000..b7390c5c6979 --- /dev/null +++ b/spec/fixtures/config/queues_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/queues.redis.sock +test: + url: unix:/path/to/queues.redis.sock +production: + url: unix:/path/to/queues.redis.sock diff --git a/spec/fixtures/config/queues_old_format_host.yml b/spec/fixtures/config/queues_old_format_host.yml new file mode 100644 index 000000000000..2dbc4c35d465 --- /dev/null +++ b/spec/fixtures/config/queues_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6381/99 +test: redis://:mypassword@localhost:6381/99 +production: redis://:mypassword@localhost:6381/99 diff --git a/spec/fixtures/config/queues_old_format_socket.yml b/spec/fixtures/config/queues_old_format_socket.yml new file mode 100644 index 000000000000..96e4eb1e33f0 --- /dev/null +++ b/spec/fixtures/config/queues_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/queues.redis.sock +test: unix:/path/to/old/queues.redis.sock +production: unix:/path/to/old/queues.redis.sock diff --git a/spec/fixtures/config/redis_config_with_env.yml b/spec/fixtures/config/redis_config_with_env.yml deleted file mode 100644 index f5860f37e475..000000000000 --- a/spec/fixtures/config/redis_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_REDIS_URL'] %> diff --git a/spec/fixtures/config/redis_new_format_socket.yml b/spec/fixtures/config/redis_new_format_socket.yml deleted file mode 100644 index 4e76830c281d..000000000000 --- a/spec/fixtures/config/redis_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/redis.sock -test: - url: unix:/path/to/redis.sock -production: - url: unix:/path/to/redis.sock diff --git a/spec/fixtures/config/redis_old_format_host.yml b/spec/fixtures/config/redis_old_format_host.yml deleted file mode 100644 index 253d0a994f5e..000000000000 --- a/spec/fixtures/config/redis_old_format_host.yml +++ /dev/null @@ -1,5 +0,0 @@ -# redis://[:password@]host[:port][/db-number][?option=value] -# more details: http://www.iana.org/assignments/uri-schemes/prov/redis -development: redis://:mypassword@localhost:6379/99 -test: redis://:mypassword@localhost:6379/99 -production: redis://:mypassword@localhost:6379/99 diff --git a/spec/fixtures/config/redis_old_format_socket.yml b/spec/fixtures/config/redis_old_format_socket.yml deleted file mode 100644 index fd31ce8ea3d7..000000000000 --- a/spec/fixtures/config/redis_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/redis.sock -test: unix:/path/to/old/redis.sock -production: unix:/path/to/old/redis.sock diff --git a/spec/fixtures/config/shared_state_config_with_env.yml b/spec/fixtures/config/shared_state_config_with_env.yml new file mode 100644 index 000000000000..2d501cd0a540 --- /dev/null +++ b/spec/fixtures/config/shared_state_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_SHARED_STATE_URL'] %> diff --git a/spec/fixtures/config/shared_state_new_format_host.yml b/spec/fixtures/config/shared_state_new_format_host.yml new file mode 100644 index 000000000000..2aea4fbf3e98 --- /dev/null +++ b/spec/fixtures/config/shared_state_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6382/99 + sentinels: + - + host: localhost + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6382/99 + sentinels: + - + host: localhost + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6382/99 + sentinels: + - + host: slave1 + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/shared_state_new_format_socket.yml b/spec/fixtures/config/shared_state_new_format_socket.yml new file mode 100644 index 000000000000..14d2c4b91692 --- /dev/null +++ b/spec/fixtures/config/shared_state_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/shared_state.redis.sock +test: + url: unix:/path/to/shared_state.redis.sock +production: + url: unix:/path/to/shared_state.redis.sock diff --git a/spec/fixtures/config/shared_state_old_format_host.yml b/spec/fixtures/config/shared_state_old_format_host.yml new file mode 100644 index 000000000000..cc922b0c2863 --- /dev/null +++ b/spec/fixtures/config/shared_state_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6382/99 +test: redis://:mypassword@localhost:6382/99 +production: redis://:mypassword@localhost:6382/99 diff --git a/spec/fixtures/config/shared_state_old_format_socket.yml b/spec/fixtures/config/shared_state_old_format_socket.yml new file mode 100644 index 000000000000..afa2a507b890 --- /dev/null +++ b/spec/fixtures/config/shared_state_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/shared_state.redis.sock +test: unix:/path/to/old/shared_state.redis.sock +production: unix:/path/to/old/shared_state.redis.sock diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 54c5ba57bdf9..8877336ae75e 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -63,7 +63,7 @@ end end - describe "#project_list_cache_key", redis: true do + describe "#project_list_cache_key", gitlab_shared_state: true do let(:project) { create(:project) } it "includes the route" do diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb index 94dcddcc30c3..181a5eec871c 100644 --- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb +++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Auth::UniqueIpsLimiter, :redis, lib: true do +describe Gitlab::Auth::UniqueIpsLimiter, :gitlab_shared_state, lib: true do include_context 'unique ips sign in limit' let(:user) { create(:user) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index b386852b1968..65e1fd129045 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :caching do let(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } let(:cache_key) { "projects/#{project.id}/pipeline_status" } @@ -28,8 +28,8 @@ expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) end - describe 'without a status in redis' do - it 'loads the status from a commit when it was not in redis' do + describe 'without a status in caching' do + it 'loads the status from a commit when it was not in caching' do empty_status = { sha: nil, status: nil, ref: nil } fake_pipeline = described_class.new( project_without_status, @@ -48,9 +48,9 @@ described_class.load_in_batch_for_projects([project_without_status]) end - it 'only connects to redis twice' do + it 'only connects to caching twice' do # Once to load, once to store in the cache - expect(Gitlab::Redis).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Caching).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -58,10 +58,10 @@ end end - describe 'when a status was cached in redis' do + describe 'when a status was cached in caching' do before do - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, + Gitlab::Caching.with do |caching| + caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end @@ -76,8 +76,8 @@ expect(pipeline_status.ref).to eq(ref) end - it 'only connects to redis once' do - expect(Gitlab::Redis).to receive(:with).exactly(1).and_call_original + it 'only connects to caching once' do + expect(Gitlab::Caching).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -94,9 +94,9 @@ end describe '.cached_results_for_projects' do - it 'loads a status from redis for all projects' do - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + it 'loads a status from caching for all projects' do + Gitlab::Caching.with do |caching| + caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end result = [{ loaded_from_cache: false, pipeline_info: { sha: nil, status: nil, ref: nil } }, @@ -203,40 +203,40 @@ end end - describe "#store_in_cache", :redis do - it "sets the object in redis" do + describe "#store_in_cache", :caching do + it "sets the object in caching" do pipeline_status.sha = '123456' pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') end end - describe '#store_in_cache_if_needed', :redis do + describe '#store_in_cache_if_needed', :caching do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil expect(ref).not_to be_nil end - it "doesn't store the status in redis when the sha is not the head of the project" do + it "doesn't store the status in caching when the sha is not the head of the project" do other_status = described_class.new( project, pipeline_info: { sha: "123456", status: "failed" } ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,8 +244,8 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, + Gitlab::Caching.with do |caching| + caching.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Caching.with { |caching| caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -263,20 +263,20 @@ end end - describe "with a status in redis", :redis do + describe "with a status in caching", :caching do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:ref) { 'master' } before do - Gitlab::Redis.with do |redis| - redis.mapped_hmset(cache_key, + Gitlab::Caching.with do |caching| + caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end describe '#load_from_cache' do - it 'reads the status from redis' do + it 'reads the status from caching' do pipeline_status.load_from_cache expect(pipeline_status.sha).to eq(sha) @@ -292,10 +292,10 @@ end describe '#delete_from_cache' do - it 'deletes values from redis' do + it 'deletes values from caching' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis.with { |redis| redis.exists(cache_key) } + key_exists = Gitlab::Caching.with { |caching| caching.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/caching_spec.rb b/spec/lib/gitlab/caching_spec.rb new file mode 100644 index 000000000000..f132f45b0b97 --- /dev/null +++ b/spec/lib/gitlab/caching_spec.rb @@ -0,0 +1,211 @@ +require 'spec_helper' + +describe Gitlab::Caching do + include StubENV + + let(:config) { 'config/caching.yml' } + + before(:each) do + stub_env('GITLAB_CACHING_CONFIG_FILE', Rails.root.join(config).to_s) + clear_raw_config + end + + after(:each) do + clear_raw_config + end + + describe '.params' do + subject { described_class.params } + + it 'withstands mutation' do + params1 = described_class.params + params2 = described_class.params + params1[:foo] = :bar + + expect(params2).not_to have_key(:foo) + end + + context 'when url contains unix socket reference' do + let(:config_old) { 'spec/fixtures/config/caching_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/caching_new_format_socket.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/old/caching.redis.sock') + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/caching.redis.sock') + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + let(:config_old) { 'spec/fixtures/config/caching_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/caching_new_format_host.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6380, db: 99) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6380, db: 99) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' + + expect(url2).not_to end_with('foobar') + end + + context 'when yml file with env variable' do + let(:config) { 'spec/fixtures/config/caching_config_with_env.yml' } + + before do + stub_env('TEST_GITLAB_CACHING_URL', 'redis://redishost:6380') + end + + it 'reads caching url from env variable' do + expect(described_class.url).to eq 'redis://redishost:6380' + end + end + end + + describe '._raw_config' do + subject { described_class._raw_config } + let(:config) { '/var/empty/doesnotexist' } + + it 'should be frozen' do + expect(subject).to be_frozen + end + + it 'returns false when the file does not exist' do + expect(subject).to eq(false) + end + end + + describe '.with' do + before { clear_pool } + after { clear_pool } + + context 'when running not on sidekiq workers' do + before { allow(Sidekiq).to receive(:server?).and_return(false) } + + it 'instantiates a connection pool with size 5' do + expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original + + described_class.with { |_caching| true } + end + end + + context 'when running on sidekiq workers' do + before do + allow(Sidekiq).to receive(:server?).and_return(true) + allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) + end + + it 'instantiates a connection pool with a size based on the concurrency of the worker' do + expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original + + described_class.with { |_caching| true } + end + end + end + + describe '#sentinels' do + subject { described_class.new(Rails.env).sentinels } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/caching_new_format_host.yml' } + + it 'returns an array of hashes with host and port keys' do + is_expected.to include(host: 'localhost', port: 26380) + is_expected.to include(host: 'slave2', port: 26380) + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/caching_old_format_host.yml' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#sentinels?' do + subject { described_class.new(Rails.env).sentinels? } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/caching_new_format_host.yml' } + + it 'returns true' do + is_expected.to be_truthy + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/caching_old_format_host.yml' } + + it 'returns false' do + is_expected.to be_falsey + end + end + end + + describe '#raw_config_hash' do + it 'returns default caching url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Caching::DEFAULT_CACHING_URL) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { 'redis://myredis:6380' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6380') + end + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(described_class).to receive(:_raw_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end + + def clear_raw_config + described_class.remove_instance_variable(:@_raw_config) + rescue NameError + # raised if @_raw_config was not set; ignore + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end +end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index c796c98ec9f0..fbe7f7528829 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -20,14 +20,14 @@ expect(current_application_settings).to be_a(ApplicationSetting) end - it 'falls back to DB if Redis returns an empty value' do + it 'falls back to DB if Caching returns an empty value' do expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) end - it 'falls back to DB if Redis fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Redis::BaseError) + it 'falls back to DB if Caching fails' do + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Caching::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb index a366d68a1460..8f93c37b4444 100644 --- a/spec/lib/gitlab/exclusive_lease_spec.rb +++ b/spec/lib/gitlab/exclusive_lease_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::ExclusiveLease, type: :redis do +describe Gitlab::ExclusiveLease, type: :gitlab_shared_state do let(:unique_key) { SecureRandom.hex(10) } describe '#try_obtain' do diff --git a/spec/lib/gitlab/health_checks/caching_check_spec.rb b/spec/lib/gitlab/health_checks/caching_check_spec.rb new file mode 100644 index 000000000000..2f22a41b23dc --- /dev/null +++ b/spec/lib/gitlab/health_checks/caching_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::CachingCheck do + include_examples 'simple_check', 'caching_ping', 'Caching', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/queues_check_spec.rb b/spec/lib/gitlab/health_checks/queues_check_spec.rb new file mode 100644 index 000000000000..1409b476d043 --- /dev/null +++ b/spec/lib/gitlab/health_checks/queues_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::QueuesCheck do + include_examples 'simple_check', 'queues_ping', 'Queues', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis_check_spec.rb deleted file mode 100644 index 734cdcb893e5..000000000000 --- a/spec/lib/gitlab/health_checks/redis_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative './simple_check_shared' - -describe Gitlab::HealthChecks::RedisCheck do - include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' -end diff --git a/spec/lib/gitlab/health_checks/shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/shared_state_check_spec.rb new file mode 100644 index 000000000000..8f506971273b --- /dev/null +++ b/spec/lib/gitlab/health_checks/shared_state_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::SharedStateCheck do + include_examples 'simple_check', 'shared_state_ping', 'SharedState', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/simple_check_shared.rb b/spec/lib/gitlab/health_checks/simple_check_shared.rb index 3f871d660344..1abebeac4ddd 100644 --- a/spec/lib/gitlab/health_checks/simple_check_shared.rb +++ b/spec/lib/gitlab/health_checks/simple_check_shared.rb @@ -47,7 +47,7 @@ allow(described_class).to receive(:check).and_return 'error!' end - it { is_expected.to have_attributes(success: false, message: "unexpected #{check_name} check result: error!") } + it { is_expected.to have_attributes(success: false, message: "unexpected #{described_class.human_name} check result: error!") } end context 'Check is timeouting' do @@ -55,7 +55,7 @@ allow(described_class).to receive(:check ).and_return Timeout::Error.new end - it { is_expected.to have_attributes(success: false, message: "#{check_name} check timed out") } + it { is_expected.to have_attributes(success: false, message: "#{described_class.human_name} check timed out") } end end diff --git a/spec/lib/gitlab/redis_spec.rb b/spec/lib/gitlab/queues_spec.rb similarity index 74% rename from spec/lib/gitlab/redis_spec.rb rename to spec/lib/gitlab/queues_spec.rb index 8b77c925705b..aef0d1197b9a 100644 --- a/spec/lib/gitlab/redis_spec.rb +++ b/spec/lib/gitlab/queues_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::Redis do +describe Gitlab::Queues do include StubENV - let(:config) { 'config/resque.yml' } + let(:config) { 'config/queues.yml' } before(:each) do - stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_QUEUES_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/queues_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/queues_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.sock') + is_expected.to include(path: '/path/to/old/queues.redis.sock') is_expected.not_to have_key(:url) end end @@ -42,21 +42,21 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.sock') + is_expected.to include(path: '/path/to/queues.redis.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/queues_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/queues_new_format_host.yml' } context 'with old format' do let(:config) { config_old } it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6381, db: 99) is_expected.not_to have_key(:url) end end @@ -65,7 +65,7 @@ let(:config) { config_new } it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6381, db: 99) is_expected.not_to have_key(:url) end end @@ -82,14 +82,14 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/queues_config_with_env.yml' } before do - stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379') + stub_env('TEST_GITLAB_QUEUES_URL', 'redis://redishost:6381') end - it 'reads redis url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6379' + it 'reads queues url from env variable' do + expect(described_class.url).to eq 'redis://redishost:6381' end end end @@ -117,7 +117,7 @@ it 'instantiates a connection pool with size 5' do expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - described_class.with { |_redis| true } + described_class.with { |_queues| true } end end @@ -130,7 +130,7 @@ it 'instantiates a connection pool with a size based on the concurrency of the worker' do expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - described_class.with { |_redis| true } + described_class.with { |_queues| true } end end end @@ -139,16 +139,16 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/queues_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26380) + is_expected.to include(host: 'localhost', port: 26381) is_expected.to include(host: 'slave2', port: 26381) end end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/queues_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/queues_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/queues_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -177,15 +177,15 @@ end describe '#raw_config_hash' do - it 'returns default redis url when no config file is present' do + it 'returns default queues url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::DEFAULT_REDIS_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Queues::DEFAULT_QUEUES_URL) end it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') + expect(subject).to receive(:fetch_config) { 'redis://myredis:6381' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6381') end end diff --git a/spec/lib/gitlab/shared_state_spec.rb b/spec/lib/gitlab/shared_state_spec.rb new file mode 100644 index 000000000000..f9a5a1894cc0 --- /dev/null +++ b/spec/lib/gitlab/shared_state_spec.rb @@ -0,0 +1,211 @@ +require 'spec_helper' + +describe Gitlab::SharedState do + include StubENV + + let(:config) { 'config/shared_state.yml' } + + before(:each) do + stub_env('GITLAB_SHARED_STATE_CONFIG_FILE', Rails.root.join(config).to_s) + clear_raw_config + end + + after(:each) do + clear_raw_config + end + + describe '.params' do + subject { described_class.params } + + it 'withstands mutation' do + params1 = described_class.params + params2 = described_class.params + params1[:foo] = :bar + + expect(params2).not_to have_key(:foo) + end + + context 'when url contains unix socket reference' do + let(:config_old) { 'spec/fixtures/config/shared_state_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/shared_state_new_format_socket.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/old/shared_state.redis.sock') + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/shared_state.redis.sock') + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + let(:config_old) { 'spec/fixtures/config/shared_state_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6382, db: 99) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6382, db: 99) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' + + expect(url2).not_to end_with('foobar') + end + + context 'when yml file with env variable' do + let(:config) { 'spec/fixtures/config/shared_state_config_with_env.yml' } + + before do + stub_env('TEST_GITLAB_SHARED_STATE_URL', 'redis://redishost:6382') + end + + it 'reads shared_state url from env variable' do + expect(described_class.url).to eq 'redis://redishost:6382' + end + end + end + + describe '._raw_config' do + subject { described_class._raw_config } + let(:config) { '/var/empty/doesnotexist' } + + it 'should be frozen' do + expect(subject).to be_frozen + end + + it 'returns false when the file does not exist' do + expect(subject).to eq(false) + end + end + + describe '.with' do + before { clear_pool } + after { clear_pool } + + context 'when running not on sidekiq workers' do + before { allow(Sidekiq).to receive(:server?).and_return(false) } + + it 'instantiates a connection pool with size 5' do + expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original + + described_class.with { |_shared_state| true } + end + end + + context 'when running on sidekiq workers' do + before do + allow(Sidekiq).to receive(:server?).and_return(true) + allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) + end + + it 'instantiates a connection pool with a size based on the concurrency of the worker' do + expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original + + described_class.with { |_shared_state| true } + end + end + end + + describe '#sentinels' do + subject { described_class.new(Rails.env).sentinels } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + + it 'returns an array of hashes with host and port keys' do + is_expected.to include(host: 'localhost', port: 26382) + is_expected.to include(host: 'slave2', port: 26382) + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/shared_state_old_format_host.yml' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#sentinels?' do + subject { described_class.new(Rails.env).sentinels? } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + + it 'returns true' do + is_expected.to be_truthy + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/shared_state_old_format_host.yml' } + + it 'returns false' do + is_expected.to be_falsey + end + end + end + + describe '#raw_config_hash' do + it 'returns default shared_state url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::SharedState::DEFAULT_SHARED_STATE_URL) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { 'redis://myredis:6382' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6382') + end + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(described_class).to receive(:_raw_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end + + def clear_raw_config + described_class.remove_instance_variable(:@_raw_config) + rescue NameError + # raised if @_raw_config was not set; ignore + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end +end diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index 496e50fbae4b..30334485130c 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::SidekiqStatus do - describe '.set', :redis do + describe '.set', :gitlab_shared_state do it 'stores the job ID' do described_class.set('123') @@ -14,7 +14,7 @@ end end - describe '.unset', :redis do + describe '.unset', :gitlab_shared_state do it 'removes the job ID' do described_class.set('123') described_class.unset('123') @@ -27,7 +27,7 @@ end end - describe '.all_completed?', :redis do + describe '.all_completed?', :gitlab_shared_state do it 'returns true if all jobs have been completed' do expect(described_class.all_completed?(%w(123))).to eq(true) end @@ -39,7 +39,7 @@ end end - describe '.num_running', :redis do + describe '.num_running', :gitlab_shared_state do it 'returns 0 if all jobs have been completed' do expect(described_class.num_running(%w(123))).to eq(0) end @@ -52,7 +52,7 @@ end end - describe '.num_completed', :redis do + describe '.num_completed', :gitlab_shared_state do it 'returns 1 if all jobs have been completed' do expect(described_class.num_completed(%w(123))).to eq(1) end @@ -74,7 +74,7 @@ end end - describe 'completed', :redis do + describe 'completed', :gitlab_shared_state do it 'returns the completed job' do expect(described_class.completed_jids(%w(123))).to eq(['123']) end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index 187d88c8c588..50d34c86c289 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -1,28 +1,28 @@ require 'spec_helper' -describe Gitlab::UserActivities, :redis, lib: true do +describe Gitlab::UserActivities, :gitlab_shared_state, lib: true do let(:now) { Time.now } describe '.record' do context 'with no time given' do - it 'uses Time.now and records an activity in Redis' do + it 'uses Time.now and records an activity in SharedState' do Timecop.freeze do now # eager-load now described_class.record(42) end - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end context 'with a time given' do - it 'uses the given time and records an activity in Redis' do + it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -31,31 +31,31 @@ describe '.delete' do context 'with a single key' do context 'and key exists' do - it 'removes the pair from Redis' do + it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and key does not exist' do - it 'removes the pair from Redis' do - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + it 'removes the pair from SharedState' do + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -63,34 +63,34 @@ context 'with multiple keys' do context 'and all keys exist' do - it 'removes the pair from Redis' do + it 'removes the pair from SharedState' do described_class.record(41, now) described_class.record(42, now) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and some keys does not exist' do - it 'removes the existing pair from Redis' do + it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis.with do |redis| - expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::SharedState.with do |shared_state| + expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index fdbb55fc874b..8d570d656f35 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -273,7 +273,7 @@ def call_verify(headers) end it 'set and notify' do - expect_any_instance_of(Redis).to receive(:publish) + expect_any_instance_of(::Redis).to receive(:publish) .with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value") subject @@ -307,7 +307,7 @@ def call_verify(headers) end it 'does not notify' do - expect_any_instance_of(Redis).not_to receive(:publish) + expect_any_instance_of(::Redis).not_to receive(:publish) subject end diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb index 3db57595fa6f..b15c4f49423b 100644 --- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb +++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb @@ -54,7 +54,7 @@ end end - describe '#up', :redis do + describe '#up', :gitlab_shared_state do let(:migration) { described_class.new } def job_count @@ -172,7 +172,7 @@ def pop_job end end - describe '#down', :redis do + describe '#down', :gitlab_shared_state do let(:migration) { described_class.new } def job_count diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index 1db9bc002ae9..6099cad70596 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -3,14 +3,14 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') -describe MigrateUserActivitiesToUsersLastActivityOn, :redis do +describe MigrateUserActivitiesToUsersLastActivityOn, :gitlab_shared_state do let(:migration) { described_class.new } let!(:user_active_1) { create(:user) } let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::Redis.with do |redis| - redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) + Gitlab::SharedState.with do |shared_state| + shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 76ce558eea0a..f05fc565307e 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -276,14 +276,14 @@ it 'sets a new last_update value when it is called the first time' do last_update = runner.ensure_runner_queue_value - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end it 'does not change if it is not expired and called again' do last_update = runner.ensure_runner_queue_value expect(runner.ensure_runner_queue_value).to eq(last_update) - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end context 'updates runner queue after changing editable value' do @@ -294,7 +294,7 @@ end it 'sets a new last_update value' do - expect_value_in_redis.not_to eq(last_update) + expect_value_in_queues.not_to eq(last_update) end end @@ -306,14 +306,14 @@ end it 'has an old last_update value' do - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end end - def expect_value_in_redis - Gitlab::Redis.with do |redis| + def expect_value_in_queues + Gitlab::Queues.with do |queues| runner_queue_key = runner.send(:runner_queue_key) - expect(redis.get(runner_queue_key)) + expect(queues.get(runner_queue_key)) end end end @@ -330,8 +330,8 @@ def expect_value_in_redis end it 'cleans up the queue' do - Gitlab::Redis.with do |redis| - expect(redis.get(queue_key)).to be_nil + Gitlab::Queues.with do |queues| + expect(queues.get(queue_key)).to be_nil end end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 6a15830a15cb..2049e323d66e 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1589,7 +1589,7 @@ def add_user(access) it { expect(user.nested_groups_projects).to eq([nested_project]) } end - describe '#refresh_authorized_projects', redis: true do + describe '#refresh_authorized_projects', gitlab_shared_state: true do let(:project1) { create(:empty_project) } let(:project2) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index cf232e7ff691..9a4cd15c0668 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -146,7 +146,7 @@ end end - describe "POST /internal/allowed", :redis do + describe "POST /internal/allowed", :gitlab_shared_state do context "access granted" do before do project.team << [user, :developer] diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index 4919ad198339..cf769c333ac2 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1187,7 +1187,7 @@ end end - context "user activities", :redis do + context "user activities", :gitlab_shared_state do let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 6ca3ef18fe64..46ab4921f8d4 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -257,7 +257,7 @@ end end - it 'updates the user last activity', :redis do + it 'updates the user last activity', :gitlab_shared_state do expect(user_activity(user)).to be_nil download(path, env) do |response| diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index 05176c3beaac..e120fb69a509 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -98,7 +98,7 @@ def hashed_subject expect(@payload['sub']).to eq hashed_subject end - it 'includes the time of the last authentication', :redis do + it 'includes the time of the last authentication', :gitlab_shared_state do expect(@payload['auth_time']).to eq user.current_sign_in_at.to_i end diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb index b06cefe071d8..939d71296a36 100644 --- a/spec/services/event_create_service_spec.rb +++ b/spec/services/event_create_service_spec.rb @@ -113,7 +113,7 @@ end end - describe '#push', :redis do + describe '#push', :gitlab_shared_state do let(:project) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index ab06f45dbb90..ee912ffa32e9 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -525,14 +525,18 @@ let(:housekeeping) { Projects::HousekeepingService.new(project) } before do - # Flush any raw Redis data stored by the housekeeping code. - Gitlab::Redis.with { |conn| conn.flushall } + # Flush any raw key-value data stored by the housekeeping code. + Gitlab::Caching.with { |conn| conn.flushall } + Gitlab::Queues.with { |conn| conn.flushall } + Gitlab::SharedState.with { |conn| conn.flushall } allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping) end after do - Gitlab::Redis.with { |conn| conn.flushall } + Gitlab::Caching.with { |conn| conn.flushall } + Gitlab::Queues.with { |conn| conn.flushall } + Gitlab::SharedState.with { |conn| conn.flushall } end it 'does not perform housekeeping when not needed' do diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 4b8589b27365..3c6ada04154e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Redis::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::SharedState::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Redis::CannotConnectError + rescue Gitlab::SharedState::CannotConnectError end end diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb index 8d67ebe3231e..3dd819d66511 100644 --- a/spec/services/users/activity_service_spec.rb +++ b/spec/services/users/activity_service_spec.rb @@ -7,7 +7,7 @@ subject(:service) { described_class.new(user, 'type') } - describe '#execute', :redis do + describe '#execute', :gitlab_shared_state do context 'when last activity is nil' do before do service.execute diff --git a/spec/services/users/refresh_authorized_projects_service_spec.rb b/spec/services/users/refresh_authorized_projects_service_spec.rb index b19374ef1a2d..c7f56dc0f17b 100644 --- a/spec/services/users/refresh_authorized_projects_service_spec.rb +++ b/spec/services/users/refresh_authorized_projects_service_spec.rb @@ -10,7 +10,7 @@ def create_authorization(project, user, access_level = Gitlab::Access::MASTER) create!(project: project, user: user, access_level: access_level) end - describe '#execute', :redis do + describe '#execute', :gitlab_shared_state do it 'refreshes the authorizations using a lease' do expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain). and_return('foo') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a58f4e664b71..8f3b77b754e7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,7 @@ require './spec/simplecov_env' +require './lib/gitlab/caching' +require './lib/gitlab/queues' +require './lib/gitlab/shared_state' SimpleCovEnv.start! ENV["RAILS_ENV"] ||= 'test' @@ -80,17 +83,21 @@ config.around(:each, :caching) do |example| caching_store = Rails.cache Rails.cache = ActiveSupport::Cache::MemoryStore.new if example.metadata[:caching] + Gitlab::Caching.with(&:flushall) + example.run + + Gitlab::Caching.with(&:flushall) Rails.cache = caching_store end - config.around(:each, :redis) do |example| - Gitlab::Redis.with(&:flushall) + config.around(:each, :gitlab_shared_state) do |example| + Gitlab::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) example.run - Gitlab::Redis.with(&:flushall) + Gitlab::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) end end diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 7cf5a65eeedc..7a69ac34688a 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,7 +1,9 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::Redis.with(&:flushall) + Gitlab::Caching.with(&:flushall) + Gitlab::Queues.with(&:flushall) + Gitlab::SharedState.with(&:flushall) end before do diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb index e583c3203aa0..b8e8debe62f7 100644 --- a/spec/workers/schedule_update_user_activity_worker_spec.rb +++ b/spec/workers/schedule_update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ScheduleUpdateUserActivityWorker, :redis do +describe ScheduleUpdateUserActivityWorker, :gitlab_shared_state do let(:now) { Time.now } before do diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb index 43e9511f1165..44ecb437e6af 100644 --- a/spec/workers/update_user_activity_worker_spec.rb +++ b/spec/workers/update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe UpdateUserActivityWorker, :redis do +describe UpdateUserActivityWorker, :gitlab_shared_state do let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) } let(:user_active_yesterday_1) { create(:user) } let(:user_active_yesterday_2) { create(:user) } @@ -25,7 +25,7 @@ end end - it 'deletes the pairs from Redis' do + it 'deletes the pairs from SharedState' do data.each { |id, time| Gitlab::UserActivities.record(id, time) } subject.perform(data) -- GitLab From cc62554b83eb68ba22a19165a11780fbafe31dd5 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 24 May 2017 19:24:12 +0800 Subject: [PATCH 02/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- scripts/prepare_build.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index db331e112936..92c0d4c875fe 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -12,9 +12,6 @@ fi # gems could not be found under some circumstance. No idea why, hours wasted. retry gem install knapsack fog-aws mime-types -cp config/resque.yml.example config/resque.yml -sed -i 's/localhost/redis/g' config/resque.yml - cp config/gitlab.yml.example config/gitlab.yml # Determine the database by looking at the job name. @@ -55,16 +52,6 @@ sed -i 's/localhost/redis/g' config/shared_state.yml # notes for when CI supports service configuration on shared image #sed -i 's/localhost/shared_state/g' config/shared_state.yml -cp config/gitlab.yml.example config/gitlab.yml - -if [ "$USE_BUNDLE_INSTALL" != "false" ]; then - bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check -fi - -# Only install knapsack after bundle install! Otherwise oddly some native -# gems could not be found under some circumstance. No idea why, hours wasted. -retry gem install knapsack fog-aws mime-types - if [ "$SETUP_DB" != "false" ]; then bundle exec rake db:drop db:create db:schema:load db:migrate -- GitLab From 9791c8e53f328c4475e691c60b8f0aa4a5ca08ef Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 25 May 2017 04:16:52 +0800 Subject: [PATCH 03/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- spec/models/user_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 2049e323d66e..9acefdd385a8 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -344,7 +344,7 @@ end end - describe '#update_tracked_fields!', :redis do + describe '#update_tracked_fields!', :gitlab_shared_state do let(:request) { OpenStruct.new(remote_ip: "127.0.0.1") } let(:user) { create(:user) } -- GitLab From 9047ddc8234e383f95fa11e910b55e76cdab59ff Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 26 May 2017 03:59:50 +0800 Subject: [PATCH 04/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- .gitignore | 6 +-- .gitlab-ci.yml | 4 +- app/controllers/health_controller.rb | 6 +-- app/models/ci/runner.rb | 4 +- app/models/project.rb | 8 ++-- config/application.rb | 10 ++-- config/initializers/7_redis.rb | 6 +-- config/initializers/session_store.rb | 4 +- config/initializers/sidekiq.rb | 6 +-- config/mail_room.yml | 4 +- ....yml.example => redis.caching.yml.example} | 2 +- ...s.yml.example => redis.queues.yml.example} | 2 +- ...example => redis.shared_state.yml.example} | 2 +- ...er_activities_to_users_last_activity_on.rb | 4 +- .../operations/cleaning_up_redis_sessions.md | 2 +- lib/gitlab/auth/unique_ips_limiter.rb | 4 +- .../cache/ci/project_pipeline_status.rb | 24 +++++----- lib/gitlab/chat_name_token.rb | 14 +++--- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/etag_caching/store.rb | 8 ++-- lib/gitlab/exclusive_lease.rb | 16 +++---- ...queues_check.rb => redis_caching_check.rb} | 6 +-- ...caching_check.rb => redis_queues_check.rb} | 6 +-- ...e_check.rb => redis_shared_state_check.rb} | 6 +-- lib/gitlab/lfs_token.rb | 8 ++-- lib/gitlab/mail_room.rb | 10 ++-- lib/gitlab/{caching.rb => redis_caching.rb} | 4 +- lib/gitlab/{queues.rb => redis_queues.rb} | 4 +- ...{shared_state.rb => redis_shared_state.rb} | 4 +- lib/gitlab/redis_wrapper.rb | 1 - lib/gitlab/user_activities.rb | 12 ++--- lib/gitlab/workhorse.rb | 8 ++-- lib/tasks/cache.rake | 6 +-- locale/gitlab.pot | 4 +- scripts/prepare_build.sh | 18 +++---- spec/config/mail_room_spec.rb | 16 +++---- spec/controllers/health_controller_spec.rb | 30 ++++++------ spec/controllers/sessions_controller_spec.rb | 2 +- spec/features/dashboard/projects_spec.rb | 2 +- spec/features/groups/members/sorting_spec.rb | 4 +- spec/features/login_spec.rb | 4 +- .../features/projects/members/sorting_spec.rb | 4 +- .../config/caching_config_with_env.yml | 2 - .../config/caching_new_format_socket.yml | 6 --- .../config/caching_old_format_socket.yml | 3 -- .../config/queues_config_with_env.yml | 2 - .../config/queues_new_format_socket.yml | 6 --- .../config/queues_old_format_socket.yml | 3 -- .../config/redis_caching_config_with_env.yml | 2 + ....yml => redis_caching_new_format_host.yml} | 0 .../redis_caching_new_format_socket.yml | 6 +++ ....yml => redis_caching_old_format_host.yml} | 0 .../redis_caching_old_format_socket.yml | 3 ++ .../config/redis_queues_config_with_env.yml | 2 + ...t.yml => redis_queues_new_format_host.yml} | 0 .../config/redis_queues_new_format_socket.yml | 6 +++ ...t.yml => redis_queues_old_format_host.yml} | 0 .../config/redis_queues_old_format_socket.yml | 3 ++ .../redis_shared_state_config_with_env.yml | 2 + ...=> redis_shared_state_new_format_host.yml} | 0 .../redis_shared_state_new_format_socket.yml | 6 +++ ...=> redis_shared_state_old_format_host.yml} | 0 .../redis_shared_state_old_format_socket.yml | 3 ++ .../config/shared_state_config_with_env.yml | 2 - .../config/shared_state_new_format_socket.yml | 6 --- .../config/shared_state_old_format_socket.yml | 3 -- spec/helpers/projects_helper_spec.rb | 2 +- .../gitlab/auth/unique_ips_limiter_spec.rb | 2 +- .../cache/ci/project_pipeline_status_spec.rb | 48 +++++++++---------- spec/lib/gitlab/current_settings_spec.rb | 2 +- spec/lib/gitlab/exclusive_lease_spec.rb | 2 +- .../health_checks/caching_check_spec.rb | 6 --- .../gitlab/health_checks/queues_check_spec.rb | 6 --- .../health_checks/redis_caching_check_spec.rb | 6 +++ .../health_checks/redis_queues_check_spec.rb | 6 +++ .../redis_shared_state_check_spec.rb | 6 +++ .../health_checks/shared_state_check_spec.rb | 6 --- ...{caching_spec.rb => redis_caching_spec.rb} | 32 ++++++------- .../{queues_spec.rb => redis_queues_spec.rb} | 32 ++++++------- ...ate_spec.rb => redis_shared_state_spec.rb} | 40 ++++++++-------- spec/lib/gitlab/sidekiq_status_spec.rb | 12 ++--- spec/lib/gitlab/user_activities_spec.rb | 42 ++++++++-------- ...migrate_process_commit_worker_jobs_spec.rb | 4 +- ...tivities_to_users_last_activity_on_spec.rb | 6 +-- spec/models/ci/runner_spec.rb | 8 ++-- spec/models/user_spec.rb | 4 +- spec/requests/api/internal_spec.rb | 2 +- spec/requests/api/users_spec.rb | 2 +- spec/requests/git_http_spec.rb | 2 +- spec/requests/openid_connect_spec.rb | 2 +- spec/services/event_create_service_spec.rb | 2 +- spec/services/git_push_service_spec.rb | 12 ++--- .../services/projects/destroy_service_spec.rb | 4 +- spec/services/users/activity_service_spec.rb | 2 +- ...efresh_authorized_projects_service_spec.rb | 2 +- spec/spec_helper.rb | 16 +++---- .../unique_ip_check_shared_examples.rb | 6 +-- ...hedule_update_user_activity_worker_spec.rb | 2 +- .../update_user_activity_worker_spec.rb | 2 +- 99 files changed, 345 insertions(+), 346 deletions(-) rename config/{caching.yml.example => redis.caching.yml.example} (96%) rename config/{queues.yml.example => redis.queues.yml.example} (96%) rename config/{shared_state.yml.example => redis.shared_state.yml.example} (96%) rename lib/gitlab/health_checks/{queues_check.rb => redis_caching_check.rb} (75%) rename lib/gitlab/health_checks/{caching_check.rb => redis_queues_check.rb} (76%) rename lib/gitlab/health_checks/{shared_state_check.rb => redis_shared_state_check.rb} (73%) rename lib/gitlab/{caching.rb => redis_caching.rb} (73%) rename lib/gitlab/{queues.rb => redis_queues.rb} (76%) rename lib/gitlab/{shared_state.rb => redis_shared_state.rb} (72%) delete mode 100644 spec/fixtures/config/caching_config_with_env.yml delete mode 100644 spec/fixtures/config/caching_new_format_socket.yml delete mode 100644 spec/fixtures/config/caching_old_format_socket.yml delete mode 100644 spec/fixtures/config/queues_config_with_env.yml delete mode 100644 spec/fixtures/config/queues_new_format_socket.yml delete mode 100644 spec/fixtures/config/queues_old_format_socket.yml create mode 100644 spec/fixtures/config/redis_caching_config_with_env.yml rename spec/fixtures/config/{caching_new_format_host.yml => redis_caching_new_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_caching_new_format_socket.yml rename spec/fixtures/config/{caching_old_format_host.yml => redis_caching_old_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_caching_old_format_socket.yml create mode 100644 spec/fixtures/config/redis_queues_config_with_env.yml rename spec/fixtures/config/{queues_new_format_host.yml => redis_queues_new_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_queues_new_format_socket.yml rename spec/fixtures/config/{queues_old_format_host.yml => redis_queues_old_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_queues_old_format_socket.yml create mode 100644 spec/fixtures/config/redis_shared_state_config_with_env.yml rename spec/fixtures/config/{shared_state_new_format_host.yml => redis_shared_state_new_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_shared_state_new_format_socket.yml rename spec/fixtures/config/{shared_state_old_format_host.yml => redis_shared_state_old_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_shared_state_old_format_socket.yml delete mode 100644 spec/fixtures/config/shared_state_config_with_env.yml delete mode 100644 spec/fixtures/config/shared_state_new_format_socket.yml delete mode 100644 spec/fixtures/config/shared_state_old_format_socket.yml delete mode 100644 spec/lib/gitlab/health_checks/caching_check_spec.rb delete mode 100644 spec/lib/gitlab/health_checks/queues_check_spec.rb create mode 100644 spec/lib/gitlab/health_checks/redis_caching_check_spec.rb create mode 100644 spec/lib/gitlab/health_checks/redis_queues_check_spec.rb create mode 100644 spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb delete mode 100644 spec/lib/gitlab/health_checks/shared_state_check_spec.rb rename spec/lib/gitlab/{caching_spec.rb => redis_caching_spec.rb} (80%) rename spec/lib/gitlab/{queues_spec.rb => redis_queues_spec.rb} (81%) rename spec/lib/gitlab/{shared_state_spec.rb => redis_shared_state_spec.rb} (75%) diff --git a/.gitignore b/.gitignore index 9f0e0298fb58..bf7dadf5cc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,9 +30,9 @@ eslint-report.html /config/initializers/smtp_settings.rb /config/initializers/relative_url.rb /config/resque.yml -/config/caching.yml -/config/queues.yml -/config/shared_state.yml +/config/redis.caching.yml +/config/redis.queues.yml +/config/redis.shared_state.yml /config/unicorn.rb /config/secrets.yml /config/sidekiq.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31d7ba4ae5b8..9fe2f435ff7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,7 +64,7 @@ stages: # image: redis:alpine # ports: # - "6381:6379" -# - shared_state: +# - redis_shared_state: # image: redis:alpine # ports: # - "6382:6379" @@ -83,7 +83,7 @@ stages: # image: redis:alpine # ports: # - "6381:6379" -# - shared_state: +# - redis_shared_state: # image: redis:alpine # ports: # - "6382:6379" diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index ae7fe80814c3..4c71f2d70dc4 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,9 +4,9 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::CachingCheck, - Gitlab::HealthChecks::QueuesCheck, - Gitlab::HealthChecks::SharedStateCheck, + Gitlab::HealthChecks::RedisCachingCheck, + Gitlab::HealthChecks::RedisQueuesCheck, + Gitlab::HealthChecks::RedisSharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 3b660e65a37f..4907f65e2d5a 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,8 +145,8 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::Queues.with do |queues| - queues.del(runner_queue_key) + Gitlab::RedisQueues.with do |redis_queues| + redis_queues.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index e2d52bedc295..293ba30b0158 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1282,15 +1282,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::SharedState.with { |shared_state| shared_state.get(pushes_since_gc_shared_state_key).to_i } + Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::SharedState.with { |shared_state| shared_state.incr(pushes_since_gc_shared_state_key) } + Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::SharedState.with { |shared_state| shared_state.del(pushes_since_gc_shared_state_key) } + Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) @@ -1353,7 +1353,7 @@ def cross_project_reference?(from) from && self != from end - def pushes_since_gc_shared_state_key + def pushes_since_gc_redis_shared_state_key "projects/#{id}/pushes_since_gc" end diff --git a/config/application.rb b/config/application.rb index d919214eb561..de565d2edacd 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,9 +6,9 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/caching') - require_dependency Rails.root.join('lib/gitlab/queues') - require_dependency Rails.root.join('lib/gitlab/shared_state') + require_dependency Rails.root.join('lib/gitlab/redis_caching') + require_dependency Rails.root.join('lib/gitlab/redis_queues') + require_dependency Rails.root.join('lib/gitlab/redis_shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -141,8 +141,8 @@ class Application < Rails::Application end # Use caching across all environments - caching_config_hash = Gitlab::Caching.params - caching_config_hash[:namespace] = Gitlab::Caching::CACHE_NAMESPACE + caching_config_hash = Gitlab::RedisCaching.params + caching_config_hash[:namespace] = Gitlab::RedisCaching::CACHE_NAMESPACE caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 9dfcae2b07f0..797361b5968f 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::Caching.with { nil } -Gitlab::Queues.with { nil } -Gitlab::SharedState.with { nil } +Gitlab::RedisCaching.with { nil } +Gitlab::RedisQueues.with { nil } +Gitlab::RedisSharedState.with { nil } diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 0181003160f7..22256d0ef13b 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,8 +13,8 @@ if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - sessions_config = Gitlab::SharedState.params - sessions_config[:namespace] = Gitlab::SharedState::SESSION_NAMESPACE + sessions_config = Gitlab::RedisSharedState.params + sessions_config[:namespace] = Gitlab::RedisSharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 4d3b2074aece..27e18fa960f7 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,6 +1,6 @@ # Custom Queues configuration -queues_config_hash = Gitlab::Queues.params -queues_config_hash[:namespace] = Gitlab::Queues::SIDEKIQ_NAMESPACE +queues_config_hash = Gitlab::RedisQueues.params +queues_config_hash[:namespace] = Gitlab::RedisQueues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } @@ -74,5 +74,5 @@ end end end -rescue Gitlab::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue Gitlab::RedisQueues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/config/mail_room.yml b/config/mail_room.yml index 24f34109a03b..d191400e8105 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Queues::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::RedisQueues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Queues::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::RedisQueues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/config/caching.yml.example b/config/redis.caching.yml.example similarity index 96% rename from config/caching.yml.example rename to config/redis.caching.yml.example index 5fe9b8f1adb1..8279ddd0c8cd 100644 --- a/config/caching.yml.example +++ b/config/redis.caching.yml.example @@ -20,7 +20,7 @@ test: # url: redis://localhost:6380 production: # Redis (single instance) - url: unix:/var/run/redis/caching.redis.sock + url: unix:/var/run/redis/redis.caching.sock ## # Redis + Sentinel (for HA) # diff --git a/config/queues.yml.example b/config/redis.queues.yml.example similarity index 96% rename from config/queues.yml.example rename to config/redis.queues.yml.example index 25284393d7e4..3c954c13fe76 100644 --- a/config/queues.yml.example +++ b/config/redis.queues.yml.example @@ -20,7 +20,7 @@ test: # url: redis://localhost:6381 production: # Redis (single instance) - url: unix:/var/run/redis/queues.redis.sock + url: unix:/var/run/redis/redis.queues.sock ## # Redis + Sentinel (for HA) # diff --git a/config/shared_state.yml.example b/config/redis.shared_state.yml.example similarity index 96% rename from config/shared_state.yml.example rename to config/redis.shared_state.yml.example index f1a6868aac4b..21bc791024fd 100644 --- a/config/shared_state.yml.example +++ b/config/redis.shared_state.yml.example @@ -20,7 +20,7 @@ test: # url: redis://localhost:6382 production: # Redis (single instance) - url: unix:/var/run/redis/shared_state.redis.sock + url: unix:/var/run/redis/redis.shared_state.sock ## # Redis + Sentinel (for HA) # diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb index 7152857a7247..f8e0895a53f8 100644 --- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -56,7 +56,7 @@ def users_table end def activities(from, to, page: 1) - Gitlab::SharedState.with do |redis| + Gitlab::RedisSharedState.with do |redis| redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, with_scores: true, limit: limit(page)) @@ -64,7 +64,7 @@ def activities(from, to, page: 1) end def activities_count(from, to) - Gitlab::SharedState.with do |redis| + Gitlab::RedisSharedState.with do |redis| redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) end end diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 253391604727..8fa9bf35fa4b 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -22,7 +22,7 @@ rcli() { # This example works for Omnibus installations of GitLab 7.3 or newer. For an # installation from source you will have to change the socket path and the # path to redis-cli. - sudo /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/shared_state.redis.socket "$@" + sudo /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.shared_state.socket "$@" } # test the new shell function; the response should be PONG diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 2cf3198324aa..7b82521cdd4e 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,9 +27,9 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::SharedState.with do |shared_state| + Gitlab::RedisSharedState.with do |redis_shared_state| unique_ips_count = nil - shared_state.multi do |r| + redis_shared_state.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) unique_ips_count = r.zcard(key) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index dee1a004614e..414e994c8f24 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,12 +23,12 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::Caching.with do |caching| - caching.multi do + result = Gitlab::RedisCaching.with do |redis_caching| + redis_caching.multi do projects.each do |project| cache_key = cache_key_for_project(project) - caching.exists(cache_key) - caching.hmget(cache_key, :sha, :status, :ref) + redis_caching.exists(cache_key) + redis_caching.hmget(cache_key, :sha, :status, :ref) end end end @@ -100,28 +100,28 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::Caching.with do |caching| - self.sha, self.status, self.ref = caching.hmget(cache_key, :sha, :status, :ref) + Gitlab::RedisCaching.with do |redis_caching| + self.sha, self.status, self.ref = redis_caching.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Caching.with do |caching| - caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Caching.with do |caching| - caching.del(cache_key) + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.del(cache_key) end end def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Caching.with do |caching| - caching.exists(cache_key) + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index b298fae8c275..d4a858dbdd4c 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ def initialize(token = new_token) end def get - Gitlab::SharedState.with do |shared_state| - data = shared_state.get(shared_state_key) + Gitlab::RedisSharedState.with do |redis_shared_state| + data = redis_shared_state.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::SharedState.with do |shared_state| + Gitlab::RedisSharedState.with do |redis_shared_state| params = params.to_json - shared_state.set(shared_state_key, params, ex: EXPIRY_TIME) + redis_shared_state.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::SharedState.with do |shared_state| - shared_state.del(shared_state_key) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ def new_token Devise.friendly_token(TOKEN_LENGTH) end - def shared_state_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index ea11a5d1cea6..9d07afc29a19 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::Caching::BaseError, ::Errno::ENOENT + rescue ::Gitlab::RedisCaching::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 939c16e68c21..587580102ed9 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -5,14 +5,14 @@ class Store SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::SharedState.with { |shared_state| shared_state.get(shared_state_key(key)) } + Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::SharedState.with do |shared_state| - shared_state.set(shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,7 +24,7 @@ def generate_etag SecureRandom.hex end - def shared_state_key(key) + def redis_shared_state_key(key) "#{SHARED_STATE_NAMESPACE}#{key}" end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index efbb13221db1..a2c6a2933cbd 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,17 +18,17 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::SharedState.with do |shared_state| - shared_state.eval(LUA_CANCEL_SCRIPT, keys: [shared_state_key(key)], argv: [uuid]) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.shared_state_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @shared_state_key = self.class.shared_state_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -37,15 +37,15 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::SharedState.with do |shared_state| - shared_state.set(@shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::SharedState.with do |shared_state| - shared_state.exists(@shared_state_key) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/health_checks/queues_check.rb b/lib/gitlab/health_checks/redis_caching_check.rb similarity index 75% rename from lib/gitlab/health_checks/queues_check.rb rename to lib/gitlab/health_checks/redis_caching_check.rb index 651949532415..5a56cc37246f 100644 --- a/lib/gitlab/health_checks/queues_check.rb +++ b/lib/gitlab/health_checks/redis_caching_check.rb @@ -1,13 +1,13 @@ module Gitlab module HealthChecks - class QueuesCheck + class RedisCachingCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'queues_ping' + 'redis_caching_ping' end def is_successful?(result) @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::Queues.with(&:ping) + Gitlab::RedisCaching.with(&:ping) end end end diff --git a/lib/gitlab/health_checks/caching_check.rb b/lib/gitlab/health_checks/redis_queues_check.rb similarity index 76% rename from lib/gitlab/health_checks/caching_check.rb rename to lib/gitlab/health_checks/redis_queues_check.rb index 155457403dd6..2b0d270fb587 100644 --- a/lib/gitlab/health_checks/caching_check.rb +++ b/lib/gitlab/health_checks/redis_queues_check.rb @@ -1,13 +1,13 @@ module Gitlab module HealthChecks - class CachingCheck + class RedisQueuesCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'caching_ping' + 'redis_queues_ping' end def is_successful?(result) @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::Caching.with(&:ping) + Gitlab::RedisQueues.with(&:ping) end end end diff --git a/lib/gitlab/health_checks/shared_state_check.rb b/lib/gitlab/health_checks/redis_shared_state_check.rb similarity index 73% rename from lib/gitlab/health_checks/shared_state_check.rb rename to lib/gitlab/health_checks/redis_shared_state_check.rb index 194dabc8b58e..68a5c4fce593 100644 --- a/lib/gitlab/health_checks/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis_shared_state_check.rb @@ -1,13 +1,13 @@ module Gitlab module HealthChecks - class SharedStateCheck + class RedisSharedStateCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'shared_state_ping' + 'redis_shared_state_ping' end def is_successful?(result) @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::SharedState.with(&:ping) + Gitlab::RedisSharedState.with(&:ping) end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index a1cf9daf69bb..4a837401592c 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ def initialize(actor) end def token - Gitlab::SharedState.with do |shared_state| - token = shared_state.get(shared_state_key) + Gitlab::RedisSharedState.with do |redis_shared_state| + token = redis_shared_state.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - shared_state.set(shared_state_key, token, ex: EXPIRY_TIME) + redis_shared_state.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ def actor_name private - def shared_state_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 7fd2e0da7bfc..88eb0afa91ab 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'queues' unless defined?(Gitlab::Queues) +require_relative 'redis_queues' unless defined?(Gitlab::RedisQueues) module Gitlab module MailRoom @@ -34,12 +34,12 @@ def fetch_config config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_queues = Gitlab::Queues.new(rails_env) - config[:redis_url] = gitlab_queues.url + gitlab_redis_queues = Gitlab::RedisQueues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url # @@ TODO this is too-much-information on topology in the client of an object - if gitlab_queues.sentinels? - config[:sentinels] = gitlab_queues.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/caching.rb b/lib/gitlab/redis_caching.rb similarity index 73% rename from lib/gitlab/caching.rb rename to lib/gitlab/redis_caching.rb index 84c8864d6cd0..9636c1219948 100644 --- a/lib/gitlab/caching.rb +++ b/lib/gitlab/redis_caching.rb @@ -3,7 +3,7 @@ require_relative 'redis_wrapper' module Gitlab - class Caching < ::Gitlab::RedisWrapper + class RedisCaching < ::Gitlab::RedisWrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze @@ -12,7 +12,7 @@ class << self end def config_file - ENV['GITLAB_CACHING_CONFIG_FILE'] || File.expand_path('../../config/caching.yml', __dir__) + ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || File.expand_path('../../config/redis.caching.yml', __dir__) end def default_url diff --git a/lib/gitlab/queues.rb b/lib/gitlab/redis_queues.rb similarity index 76% rename from lib/gitlab/queues.rb rename to lib/gitlab/redis_queues.rb index 661f886fecc2..3cdd9cde17cf 100644 --- a/lib/gitlab/queues.rb +++ b/lib/gitlab/redis_queues.rb @@ -3,7 +3,7 @@ require_relative 'redis_wrapper' module Gitlab - class Queues < ::Gitlab::RedisWrapper + class RedisQueues < ::Gitlab::RedisWrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze @@ -13,7 +13,7 @@ class << self end def config_file - ENV['GITLAB_QUEUES_CONFIG_FILE'] || File.expand_path('../../config/queues.yml', __dir__) + ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || File.expand_path('../../config/redis.queues.yml', __dir__) end def default_url diff --git a/lib/gitlab/shared_state.rb b/lib/gitlab/redis_shared_state.rb similarity index 72% rename from lib/gitlab/shared_state.rb rename to lib/gitlab/redis_shared_state.rb index 25381dc36611..cba6c30d5024 100644 --- a/lib/gitlab/shared_state.rb +++ b/lib/gitlab/redis_shared_state.rb @@ -3,7 +3,7 @@ require_relative 'redis_wrapper' module Gitlab - class SharedState < ::Gitlab::RedisWrapper + class RedisSharedState < ::Gitlab::RedisWrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze @@ -12,7 +12,7 @@ class << self end def config_file - ENV['GITLAB_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../config/shared_state.yml', __dir__) + ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../config/redis.shared_state.yml', __dir__) end def default_url diff --git a/lib/gitlab/redis_wrapper.rb b/lib/gitlab/redis_wrapper.rb index 88976938898e..63d247c782d5 100644 --- a/lib/gitlab/redis_wrapper.rb +++ b/lib/gitlab/redis_wrapper.rb @@ -1,5 +1,4 @@ # This file should only be used by sub-classes, not directly by any clients of the sub-classes -# This file should not have any direct dependency on Rails environment # please require all dependencies below: require 'rails' require 'redis' diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index 2366071ec03f..be5c8a707b8b 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,14 +6,14 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::SharedState.with do |shared_state| - shared_state.hset(KEY, key, time.to_i) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::SharedState.with do |shared_state| - shared_state.hdel(KEY, keys) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.hdel(KEY, keys) end end @@ -21,8 +21,8 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::SharedState.with do |shared_state| - shared_state.hscan(KEY, cursor, count: BATCH_SIZE) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.hscan(KEY, cursor, count: BATCH_SIZE) end Hash[pairs].each { |pair| yield pair } diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5e1f15fa7b7d..2a30bd6a9e70 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,13 +174,13 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Queues.with do |queues| - result = queues.set(key, value, ex: expire, nx: !overwrite) + Gitlab::RedisQueues.with do |redis_queues| + result = redis_queues.set(key, value, ex: expire, nx: !overwrite) if result - queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + redis_queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") value else - queues.get(key) + redis_queues.get(key) end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index e31011ca8ce1..0560f37937be 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,17 +1,17 @@ namespace :cache do namespace :clear do - # @@ TODO - push implementation into Gitlab::Caching + # @@ TODO - push implementation into Gitlab::RedisCaching KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear key/value cache" task gitlab_kv_cache: :environment do - Gitlab::Caching.with do |kv_cache| + Gitlab::RedisCaching.with do |kv_cache| cursor = KV_CACHE_SCAN_START_STOP loop do cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::Caching::CACHE_NAMESPACE}*", + match: "#{Gitlab::RedisCaching::CACHE_NAMESPACE}*", count: KV_CACHE_CLEAR_BATCH_SIZE ) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index a64d6b36ebf4..5a2aff5fffc9 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8,8 +8,8 @@ msgid "" msgstr "" "Project-Id-Version: gitlab 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-05-24 19:07+0800\n" -"PO-Revision-Date: 2017-05-24 19:07+0800\n" +"POT-Creation-Date: 2017-05-25 12:34+0800\n" +"PO-Revision-Date: 2017-05-25 12:34+0800\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 92c0d4c875fe..a4e3cefbcaa9 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -34,23 +34,23 @@ else # Assume it's mysql sed -i 's/# host:.*/host: mysql/g' config/database.yml fi -cp config/caching.yml.example config/caching.yml -sed -i 's/localhost/redis/g' config/caching.yml +cp config/redis.caching.yml.example config/redis.caching.yml +sed -i 's/localhost/redis/g' config/redis.caching.yml # # notes for when CI supports service configuration on shared image -#sed -i 's/localhost/caching/g' config/caching.yml +#sed -i 's/localhost/caching/g' config/redis.caching.yml -cp config/queues.yml.example config/queues.yml -sed -i 's/localhost/redis/g' config/queues.yml +cp config/redis.queues.yml.example config/redis.queues.yml +sed -i 's/localhost/redis/g' config/redis.queues.yml # # notes for when CI supports service configuration on shared image -#sed -i 's/localhost/queues/g' config/queues.yml +#sed -i 's/localhost/queues/g' config/redis.queues.yml -cp config/shared_state.yml.example config/shared_state.yml -sed -i 's/localhost/redis/g' config/shared_state.yml +cp config/redis.shared_state.yml.example config/redis.shared_state.yml +sed -i 's/localhost/redis/g' config/redis.shared_state.yml # # notes for when CI supports service configuration on shared image -#sed -i 's/localhost/shared_state/g' config/shared_state.yml +#sed -i 's/localhost/redis_shared_state/g' config/redis.shared_state.yml if [ "$SETUP_DB" != "false" ]; then bundle exec rake db:drop db:create db:schema:load db:migrate diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index a30f340f5242..e96a9d0e43b2 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -5,12 +5,12 @@ let(:mailroom_config_path) { 'config/mail_room.yml' } let(:gitlab_config_path) { 'config/mail_room.yml' } - let(:queues_config_path) { 'config/queues.yml' } + let(:queues_config_path) { 'config/redis.queues.yml' } let(:configuration) do vars = { 'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path), - 'GITLAB_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path) + 'GITLAB_REDIS_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path) } cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result" @@ -21,7 +21,7 @@ end before(:each) do - stub_env('GITLAB_QUEUES_CONFIG_FILE', absolute_path(queues_config_path)) + stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', absolute_path(queues_config_path)) clear_queues_raw_config end @@ -39,9 +39,9 @@ context 'when incoming email is enabled' do let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } - let(:queues_config_path) { 'spec/fixtures/config/queues_new_format_host.yml' } + let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - let(:gitlab_queues) { Gitlab::Queues.new(Rails.env) } + let(:gitlab_redis_queues) { Gitlab::RedisQueues.new(Rails.env) } it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) @@ -56,8 +56,8 @@ expect(mailbox[:name]).to eq('inbox') expect(mailbox[:idle_timeout]).to eq(60) - redis_url = gitlab_queues.url - sentinels = gitlab_queues.sentinels + redis_url = gitlab_redis_queues.url + sentinels = gitlab_redis_queues.sentinels expect(mailbox[:delivery_options][:redis_url]).to be_present expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) @@ -74,7 +74,7 @@ end def clear_queues_raw_config - Gitlab::Queues.remove_instance_variable(:@_raw_config) + Gitlab::RedisQueues.remove_instance_variable(:@_raw_config) rescue NameError # raised if @_raw_config was not set; ignore end diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index c73639024b41..24d39ee64685 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,9 +19,9 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') - expect(json_response['queues_check']['status']).to eq('ok') - expect(json_response['shared_state_check']['status']).to eq('ok') + expect(json_response['redis_caching_check']['status']).to eq('ok') + expect(json_response['redis_queues_check']['status']).to eq('ok') + expect(json_response['redis_shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') end @@ -44,9 +44,9 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') - expect(json_response['queues_check']['status']).to eq('ok') - expect(json_response['shared_state_check']['status']).to eq('ok') + expect(json_response['redis_caching_check']['status']).to eq('ok') + expect(json_response['redis_queues_check']['status']).to eq('ok') + expect(json_response['redis_shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') end end @@ -74,23 +74,23 @@ it 'returns Caching ping metrics' do get :metrics - expect(response.body).to match(/^caching_ping_timeout 0$/) - expect(response.body).to match(/^caching_ping_success 1$/) - expect(response.body).to match(/^caching_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^redis_caching_ping_timeout 0$/) + expect(response.body).to match(/^redis_caching_ping_success 1$/) + expect(response.body).to match(/^redis_caching_ping_latency [0-9\.]+$/) end it 'returns Queues ping metrics' do get :metrics - expect(response.body).to match(/^queues_ping_timeout 0$/) - expect(response.body).to match(/^queues_ping_success 1$/) - expect(response.body).to match(/^queues_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^redis_queues_ping_timeout 0$/) + expect(response.body).to match(/^redis_queues_ping_success 1$/) + expect(response.body).to match(/^redis_queues_ping_latency [0-9\.]+$/) end it 'returns SharedState ping metrics' do get :metrics - expect(response.body).to match(/^shared_state_ping_timeout 0$/) - expect(response.body).to match(/^shared_state_ping_success 1$/) - expect(response.body).to match(/^shared_state_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^redis_shared_state_ping_timeout 0$/) + expect(response.body).to match(/^redis_shared_state_ping_success 1$/) + expect(response.body).to match(/^redis_shared_state_ping_latency [0-9\.]+$/) end it 'returns file system check metrics' do diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index e7e59731f662..d7443a5943d0 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -16,7 +16,7 @@ end end - context 'when using valid password', :gitlab_shared_state do + context 'when using valid password', :gitlab_redis_shared_state do include UserActivitiesHelpers let(:user) { create(:user) } diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb index 0d97e7b2e639..4a853ffc75ab 100644 --- a/spec/features/dashboard/projects_spec.rb +++ b/spec/features/dashboard/projects_spec.rb @@ -14,7 +14,7 @@ expect(page).to have_content('awesome stuff') end - describe "with a pipeline", gitlab_shared_state: true do + describe "with a pipeline", gitlab_redis_shared_state: true do let!(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) } before do diff --git a/spec/features/groups/members/sorting_spec.rb b/spec/features/groups/members/sorting_spec.rb index 1f66a8b41c49..8b2735372fc6 100644 --- a/spec/features/groups/members/sorting_spec.rb +++ b/spec/features/groups/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :gitlab_shared_state do + scenario 'sorts by recent sign in', :gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(owner.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :gitlab_shared_state do + scenario 'sorts by oldest sign in', :gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb index 8490661d0253..8593ef922050 100644 --- a/spec/features/login_spec.rb +++ b/spec/features/login_spec.rb @@ -41,7 +41,7 @@ expect(page).to have_content('Your account has been blocked.') end - it 'does not update Devise trackable attributes', :gitlab_shared_state do + it 'does not update Devise trackable attributes', :gitlab_redis_shared_state do user = create(:user, :blocked) expect { login_with(user) }.not_to change { user.reload.sign_in_count } @@ -55,7 +55,7 @@ expect(page).to have_content('Invalid Login or password.') end - it 'does not update Devise trackable attributes', :gitlab_shared_state do + it 'does not update Devise trackable attributes', :gitlab_redis_shared_state do expect { login_with(User.ghost) }.not_to change { User.ghost.reload.sign_in_count } end end diff --git a/spec/features/projects/members/sorting_spec.rb b/spec/features/projects/members/sorting_spec.rb index acd38598defd..714d9cf4d63a 100644 --- a/spec/features/projects/members/sorting_spec.rb +++ b/spec/features/projects/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :gitlab_shared_state do + scenario 'sorts by recent sign in', :gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(master.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :gitlab_shared_state do + scenario 'sorts by oldest sign in', :gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/fixtures/config/caching_config_with_env.yml b/spec/fixtures/config/caching_config_with_env.yml deleted file mode 100644 index 97341956f140..000000000000 --- a/spec/fixtures/config/caching_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_CACHING_URL'] %> diff --git a/spec/fixtures/config/caching_new_format_socket.yml b/spec/fixtures/config/caching_new_format_socket.yml deleted file mode 100644 index 15b5226ce29b..000000000000 --- a/spec/fixtures/config/caching_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/caching.redis.sock -test: - url: unix:/path/to/caching.redis.sock -production: - url: unix:/path/to/caching.redis.sock diff --git a/spec/fixtures/config/caching_old_format_socket.yml b/spec/fixtures/config/caching_old_format_socket.yml deleted file mode 100644 index 67fac4877e58..000000000000 --- a/spec/fixtures/config/caching_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/caching.redis.sock -test: unix:/path/to/old/caching.redis.sock -production: unix:/path/to/old/caching.redis.sock diff --git a/spec/fixtures/config/queues_config_with_env.yml b/spec/fixtures/config/queues_config_with_env.yml deleted file mode 100644 index 7c5b2b9e69f4..000000000000 --- a/spec/fixtures/config/queues_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_QUEUES_URL'] %> diff --git a/spec/fixtures/config/queues_new_format_socket.yml b/spec/fixtures/config/queues_new_format_socket.yml deleted file mode 100644 index b7390c5c6979..000000000000 --- a/spec/fixtures/config/queues_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/queues.redis.sock -test: - url: unix:/path/to/queues.redis.sock -production: - url: unix:/path/to/queues.redis.sock diff --git a/spec/fixtures/config/queues_old_format_socket.yml b/spec/fixtures/config/queues_old_format_socket.yml deleted file mode 100644 index 96e4eb1e33f0..000000000000 --- a/spec/fixtures/config/queues_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/queues.redis.sock -test: unix:/path/to/old/queues.redis.sock -production: unix:/path/to/old/queues.redis.sock diff --git a/spec/fixtures/config/redis_caching_config_with_env.yml b/spec/fixtures/config/redis_caching_config_with_env.yml new file mode 100644 index 000000000000..04ca455a543e --- /dev/null +++ b/spec/fixtures/config/redis_caching_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_CACHING_URL'] %> diff --git a/spec/fixtures/config/caching_new_format_host.yml b/spec/fixtures/config/redis_caching_new_format_host.yml similarity index 100% rename from spec/fixtures/config/caching_new_format_host.yml rename to spec/fixtures/config/redis_caching_new_format_host.yml diff --git a/spec/fixtures/config/redis_caching_new_format_socket.yml b/spec/fixtures/config/redis_caching_new_format_socket.yml new file mode 100644 index 000000000000..9f6d4515f1c7 --- /dev/null +++ b/spec/fixtures/config/redis_caching_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.caching.sock +test: + url: unix:/path/to/redis.caching.sock +production: + url: unix:/path/to/redis.caching.sock diff --git a/spec/fixtures/config/caching_old_format_host.yml b/spec/fixtures/config/redis_caching_old_format_host.yml similarity index 100% rename from spec/fixtures/config/caching_old_format_host.yml rename to spec/fixtures/config/redis_caching_old_format_host.yml diff --git a/spec/fixtures/config/redis_caching_old_format_socket.yml b/spec/fixtures/config/redis_caching_old_format_socket.yml new file mode 100644 index 000000000000..c201cc856614 --- /dev/null +++ b/spec/fixtures/config/redis_caching_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.caching.sock +test: unix:/path/to/old/redis.caching.sock +production: unix:/path/to/old/redis.caching.sock diff --git a/spec/fixtures/config/redis_queues_config_with_env.yml b/spec/fixtures/config/redis_queues_config_with_env.yml new file mode 100644 index 000000000000..d16a9d8a7f85 --- /dev/null +++ b/spec/fixtures/config/redis_queues_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_QUEUES_URL'] %> diff --git a/spec/fixtures/config/queues_new_format_host.yml b/spec/fixtures/config/redis_queues_new_format_host.yml similarity index 100% rename from spec/fixtures/config/queues_new_format_host.yml rename to spec/fixtures/config/redis_queues_new_format_host.yml diff --git a/spec/fixtures/config/redis_queues_new_format_socket.yml b/spec/fixtures/config/redis_queues_new_format_socket.yml new file mode 100644 index 000000000000..b488d84d022a --- /dev/null +++ b/spec/fixtures/config/redis_queues_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.queues.sock +test: + url: unix:/path/to/redis.queues.sock +production: + url: unix:/path/to/redis.queues.sock diff --git a/spec/fixtures/config/queues_old_format_host.yml b/spec/fixtures/config/redis_queues_old_format_host.yml similarity index 100% rename from spec/fixtures/config/queues_old_format_host.yml rename to spec/fixtures/config/redis_queues_old_format_host.yml diff --git a/spec/fixtures/config/redis_queues_old_format_socket.yml b/spec/fixtures/config/redis_queues_old_format_socket.yml new file mode 100644 index 000000000000..53f5db727587 --- /dev/null +++ b/spec/fixtures/config/redis_queues_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.queues.sock +test: unix:/path/to/old/redis.queues.sock +production: unix:/path/to/old/redis.queues.sock diff --git a/spec/fixtures/config/redis_shared_state_config_with_env.yml b/spec/fixtures/config/redis_shared_state_config_with_env.yml new file mode 100644 index 000000000000..eab7203d0de5 --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_SHARED_STATE_URL'] %> diff --git a/spec/fixtures/config/shared_state_new_format_host.yml b/spec/fixtures/config/redis_shared_state_new_format_host.yml similarity index 100% rename from spec/fixtures/config/shared_state_new_format_host.yml rename to spec/fixtures/config/redis_shared_state_new_format_host.yml diff --git a/spec/fixtures/config/redis_shared_state_new_format_socket.yml b/spec/fixtures/config/redis_shared_state_new_format_socket.yml new file mode 100644 index 000000000000..1b0e699729eb --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.shared_state.sock +test: + url: unix:/path/to/redis.shared_state.sock +production: + url: unix:/path/to/redis.shared_state.sock diff --git a/spec/fixtures/config/shared_state_old_format_host.yml b/spec/fixtures/config/redis_shared_state_old_format_host.yml similarity index 100% rename from spec/fixtures/config/shared_state_old_format_host.yml rename to spec/fixtures/config/redis_shared_state_old_format_host.yml diff --git a/spec/fixtures/config/redis_shared_state_old_format_socket.yml b/spec/fixtures/config/redis_shared_state_old_format_socket.yml new file mode 100644 index 000000000000..4746afbb5ef5 --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.shared_state.sock +test: unix:/path/to/old/redis.shared_state.sock +production: unix:/path/to/old/redis.shared_state.sock diff --git a/spec/fixtures/config/shared_state_config_with_env.yml b/spec/fixtures/config/shared_state_config_with_env.yml deleted file mode 100644 index 2d501cd0a540..000000000000 --- a/spec/fixtures/config/shared_state_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_SHARED_STATE_URL'] %> diff --git a/spec/fixtures/config/shared_state_new_format_socket.yml b/spec/fixtures/config/shared_state_new_format_socket.yml deleted file mode 100644 index 14d2c4b91692..000000000000 --- a/spec/fixtures/config/shared_state_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/shared_state.redis.sock -test: - url: unix:/path/to/shared_state.redis.sock -production: - url: unix:/path/to/shared_state.redis.sock diff --git a/spec/fixtures/config/shared_state_old_format_socket.yml b/spec/fixtures/config/shared_state_old_format_socket.yml deleted file mode 100644 index afa2a507b890..000000000000 --- a/spec/fixtures/config/shared_state_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/shared_state.redis.sock -test: unix:/path/to/old/shared_state.redis.sock -production: unix:/path/to/old/shared_state.redis.sock diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 8877336ae75e..b424ccc82d9e 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -63,7 +63,7 @@ end end - describe "#project_list_cache_key", gitlab_shared_state: true do + describe "#project_list_cache_key", gitlab_redis_shared_state: true do let(:project) { create(:project) } it "includes the route" do diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb index 181a5eec871c..6af575f28cea 100644 --- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb +++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Auth::UniqueIpsLimiter, :gitlab_shared_state, lib: true do +describe Gitlab::Auth::UniqueIpsLimiter, :gitlab_redis_shared_state, lib: true do include_context 'unique ips sign in limit' let(:user) { create(:user) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index 65e1fd129045..a9d9db4adead 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -28,8 +28,8 @@ expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) end - describe 'without a status in caching' do - it 'loads the status from a commit when it was not in caching' do + describe 'without a status in redis_caching' do + it 'loads the status from a commit when it was not in redis_caching' do empty_status = { sha: nil, status: nil, ref: nil } fake_pipeline = described_class.new( project_without_status, @@ -48,9 +48,9 @@ described_class.load_in_batch_for_projects([project_without_status]) end - it 'only connects to caching twice' do + it 'only connects to redis_caching twice' do # Once to load, once to store in the cache - expect(Gitlab::Caching).to receive(:with).exactly(2).and_call_original + expect(Gitlab::RedisCaching).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -58,10 +58,10 @@ end end - describe 'when a status was cached in caching' do + describe 'when a status was cached in redis_caching' do before do - Gitlab::Caching.with do |caching| - caching.mapped_hmset(cache_key, + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end @@ -76,8 +76,8 @@ expect(pipeline_status.ref).to eq(ref) end - it 'only connects to caching once' do - expect(Gitlab::Caching).to receive(:with).exactly(1).and_call_original + it 'only connects to redis_caching once' do + expect(Gitlab::RedisCaching).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -95,8 +95,8 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::Caching.with do |caching| - caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end result = [{ loaded_from_cache: false, pipeline_info: { sha: nil, status: nil, ref: nil } }, @@ -183,7 +183,7 @@ end end - describe "#load_from_project" do + describe "#load_from_project", :caching do let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } it 'reads the status from the pipeline for the commit' do @@ -209,7 +209,7 @@ pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') @@ -222,21 +222,21 @@ pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil expect(ref).not_to be_nil end - it "doesn't store the status in caching when the sha is not the head of the project" do + it "doesn't store the status in redis_caching when the sha is not the head of the project" do other_status = described_class.new( project, pipeline_info: { sha: "123456", status: "failed" } ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Caching.with { |caching| caching.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,8 +244,8 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Caching.with do |caching| - caching.mapped_hmset(cache_key, + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Caching.with { |caching| caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -269,14 +269,14 @@ let(:ref) { 'master' } before do - Gitlab::Caching.with do |caching| - caching.mapped_hmset(cache_key, + Gitlab::RedisCaching.with do |redis_caching| + redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end describe '#load_from_cache' do - it 'reads the status from caching' do + it 'reads the status from redis_caching' do pipeline_status.load_from_cache expect(pipeline_status.sha).to eq(sha) @@ -292,10 +292,10 @@ end describe '#delete_from_cache' do - it 'deletes values from caching' do + it 'deletes values from redis_caching' do pipeline_status.delete_from_cache - key_exists = Gitlab::Caching.with { |caching| caching.exists(cache_key) } + key_exists = Gitlab::RedisCaching.with { |redis_caching| redis_caching.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index fbe7f7528829..bd1efdf20a3e 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Caching::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::RedisCaching::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb index 8f93c37b4444..5bfd4b25d3a9 100644 --- a/spec/lib/gitlab/exclusive_lease_spec.rb +++ b/spec/lib/gitlab/exclusive_lease_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::ExclusiveLease, type: :gitlab_shared_state do +describe Gitlab::ExclusiveLease, type: :gitlab_redis_shared_state do let(:unique_key) { SecureRandom.hex(10) } describe '#try_obtain' do diff --git a/spec/lib/gitlab/health_checks/caching_check_spec.rb b/spec/lib/gitlab/health_checks/caching_check_spec.rb deleted file mode 100644 index 2f22a41b23dc..000000000000 --- a/spec/lib/gitlab/health_checks/caching_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative './simple_check_shared' - -describe Gitlab::HealthChecks::CachingCheck do - include_examples 'simple_check', 'caching_ping', 'Caching', 'PONG' -end diff --git a/spec/lib/gitlab/health_checks/queues_check_spec.rb b/spec/lib/gitlab/health_checks/queues_check_spec.rb deleted file mode 100644 index 1409b476d043..000000000000 --- a/spec/lib/gitlab/health_checks/queues_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative './simple_check_shared' - -describe Gitlab::HealthChecks::QueuesCheck do - include_examples 'simple_check', 'queues_ping', 'Queues', 'PONG' -end diff --git a/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb b/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb new file mode 100644 index 000000000000..710028a4ec8d --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::RedisCachingCheck do + include_examples 'simple_check', 'redis_caching_ping', 'RedisCaching', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb b/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb new file mode 100644 index 000000000000..52727f5a8b34 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::RedisQueuesCheck do + include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb new file mode 100644 index 000000000000..8b34a9e07083 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative './simple_check_shared' + +describe Gitlab::HealthChecks::RedisSharedStateCheck do + include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/shared_state_check_spec.rb deleted file mode 100644 index 8f506971273b..000000000000 --- a/spec/lib/gitlab/health_checks/shared_state_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative './simple_check_shared' - -describe Gitlab::HealthChecks::SharedStateCheck do - include_examples 'simple_check', 'shared_state_ping', 'SharedState', 'PONG' -end diff --git a/spec/lib/gitlab/caching_spec.rb b/spec/lib/gitlab/redis_caching_spec.rb similarity index 80% rename from spec/lib/gitlab/caching_spec.rb rename to spec/lib/gitlab/redis_caching_spec.rb index f132f45b0b97..f373716a0b39 100644 --- a/spec/lib/gitlab/caching_spec.rb +++ b/spec/lib/gitlab/redis_caching_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::Caching do +describe Gitlab::RedisCaching do include StubENV - let(:config) { 'config/caching.yml' } + let(:config) { 'config/redis.caching.yml' } before(:each) do - stub_env('GITLAB_CACHING_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_REDIS_CACHING_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/caching_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/caching_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/caching.redis.sock') + is_expected.to include(path: '/path/to/old/redis.caching.sock') is_expected.not_to have_key(:url) end end @@ -42,15 +42,15 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/caching.redis.sock') + is_expected.to include(path: '/path/to/redis.caching.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/caching_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/caching_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } context 'with old format' do let(:config) { config_old } @@ -82,10 +82,10 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/caching_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/redis_caching_config_with_env.yml' } before do - stub_env('TEST_GITLAB_CACHING_URL', 'redis://redishost:6380') + stub_env('TEST_GITLAB_REDIS_CACHING_URL', 'redis://redishost:6380') end it 'reads caching url from env variable' do @@ -139,7 +139,7 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do is_expected.to include(host: 'localhost', port: 26380) @@ -148,7 +148,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Caching::DEFAULT_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisCaching::DEFAULT_CACHING_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/queues_spec.rb b/spec/lib/gitlab/redis_queues_spec.rb similarity index 81% rename from spec/lib/gitlab/queues_spec.rb rename to spec/lib/gitlab/redis_queues_spec.rb index aef0d1197b9a..b498f48cd236 100644 --- a/spec/lib/gitlab/queues_spec.rb +++ b/spec/lib/gitlab/redis_queues_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::Queues do +describe Gitlab::RedisQueues do include StubENV - let(:config) { 'config/queues.yml' } + let(:config) { 'config/redis.queues.yml' } before(:each) do - stub_env('GITLAB_QUEUES_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/queues_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/queues_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/redis_queues_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_queues_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/queues.redis.sock') + is_expected.to include(path: '/path/to/old/redis.queues.sock') is_expected.not_to have_key(:url) end end @@ -42,15 +42,15 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/queues.redis.sock') + is_expected.to include(path: '/path/to/redis.queues.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/queues_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/queues_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } context 'with old format' do let(:config) { config_old } @@ -82,10 +82,10 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/queues_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/redis_queues_config_with_env.yml' } before do - stub_env('TEST_GITLAB_QUEUES_URL', 'redis://redishost:6381') + stub_env('TEST_GITLAB_REDIS_QUEUES_URL', 'redis://redishost:6381') end it 'reads queues url from env variable' do @@ -139,7 +139,7 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/queues_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do is_expected.to include(host: 'localhost', port: 26381) @@ -148,7 +148,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/queues_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/queues_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/queues_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -180,7 +180,7 @@ it 'returns default queues url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Queues::DEFAULT_QUEUES_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisQueues::DEFAULT_QUEUES_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/shared_state_spec.rb b/spec/lib/gitlab/redis_shared_state_spec.rb similarity index 75% rename from spec/lib/gitlab/shared_state_spec.rb rename to spec/lib/gitlab/redis_shared_state_spec.rb index f9a5a1894cc0..449691980d46 100644 --- a/spec/lib/gitlab/shared_state_spec.rb +++ b/spec/lib/gitlab/redis_shared_state_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::SharedState do +describe Gitlab::RedisSharedState do include StubENV - let(:config) { 'config/shared_state.yml' } + let(:config) { 'config/redis.shared_state.yml' } before(:each) do - stub_env('GITLAB_SHARED_STATE_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_REDIS_SHARED_STATE_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/shared_state_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/shared_state_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/redis_shared_state_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_shared_state_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/shared_state.redis.sock') + is_expected.to include(path: '/path/to/old/redis.shared_state.sock') is_expected.not_to have_key(:url) end end @@ -42,15 +42,15 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/shared_state.redis.sock') + is_expected.to include(path: '/path/to/redis.shared_state.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/shared_state_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } context 'with old format' do let(:config) { config_old } @@ -82,13 +82,13 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/shared_state_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/redis_shared_state_config_with_env.yml' } before do - stub_env('TEST_GITLAB_SHARED_STATE_URL', 'redis://redishost:6382') + stub_env('TEST_GITLAB_REDIS_SHARED_STATE_URL', 'redis://redishost:6382') end - it 'reads shared_state url from env variable' do + it 'reads redis_shared_state url from env variable' do expect(described_class.url).to eq 'redis://redishost:6382' end end @@ -117,7 +117,7 @@ it 'instantiates a connection pool with size 5' do expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - described_class.with { |_shared_state| true } + described_class.with { |_redis_shared_state| true } end end @@ -130,7 +130,7 @@ it 'instantiates a connection pool with a size based on the concurrency of the worker' do expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - described_class.with { |_shared_state| true } + described_class.with { |_redis_shared_state| true } end end end @@ -139,7 +139,7 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do is_expected.to include(host: 'localhost', port: 26382) @@ -148,7 +148,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/shared_state_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/shared_state_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/shared_state_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -177,10 +177,10 @@ end describe '#raw_config_hash' do - it 'returns default shared_state url when no config file is present' do + it 'returns default redis_shared_state url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::SharedState::DEFAULT_SHARED_STATE_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisSharedState::DEFAULT_SHARED_STATE_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index 30334485130c..b0efba8e0ade 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::SidekiqStatus do - describe '.set', :gitlab_shared_state do + describe '.set', :gitlab_redis_shared_state do it 'stores the job ID' do described_class.set('123') @@ -14,7 +14,7 @@ end end - describe '.unset', :gitlab_shared_state do + describe '.unset', :gitlab_redis_shared_state do it 'removes the job ID' do described_class.set('123') described_class.unset('123') @@ -27,7 +27,7 @@ end end - describe '.all_completed?', :gitlab_shared_state do + describe '.all_completed?', :gitlab_redis_shared_state do it 'returns true if all jobs have been completed' do expect(described_class.all_completed?(%w(123))).to eq(true) end @@ -39,7 +39,7 @@ end end - describe '.num_running', :gitlab_shared_state do + describe '.num_running', :gitlab_redis_shared_state do it 'returns 0 if all jobs have been completed' do expect(described_class.num_running(%w(123))).to eq(0) end @@ -52,7 +52,7 @@ end end - describe '.num_completed', :gitlab_shared_state do + describe '.num_completed', :gitlab_redis_shared_state do it 'returns 1 if all jobs have been completed' do expect(described_class.num_completed(%w(123))).to eq(1) end @@ -74,7 +74,7 @@ end end - describe 'completed', :gitlab_shared_state do + describe 'completed', :gitlab_redis_shared_state do it 'returns the completed job' do expect(described_class.completed_jids(%w(123))).to eq(['123']) end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index 50d34c86c289..af774b7daac4 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::UserActivities, :gitlab_shared_state, lib: true do +describe Gitlab::UserActivities, :gitlab_redis_shared_state, lib: true do let(:now) { Time.now } describe '.record' do @@ -11,8 +11,8 @@ described_class.record(42) end - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -21,8 +21,8 @@ it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -34,28 +34,28 @@ it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and key does not exist' do it 'removes the pair from SharedState' do - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -67,14 +67,14 @@ described_class.record(41, now) described_class.record(42, now) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -83,14 +83,14 @@ it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::SharedState.with do |shared_state| - expect(shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::RedisSharedState.with do |redis_shared_state| + expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb index b15c4f49423b..131214cc7008 100644 --- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb +++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb @@ -54,7 +54,7 @@ end end - describe '#up', :gitlab_shared_state do + describe '#up', :gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count @@ -172,7 +172,7 @@ def pop_job end end - describe '#down', :gitlab_shared_state do + describe '#down', :gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index 6099cad70596..c3c1ab41259e 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -3,14 +3,14 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') -describe MigrateUserActivitiesToUsersLastActivityOn, :gitlab_shared_state do +describe MigrateUserActivitiesToUsersLastActivityOn, :gitlab_redis_shared_state do let(:migration) { described_class.new } let!(:user_active_1) { create(:user) } let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::SharedState.with do |shared_state| - shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) + Gitlab::RedisSharedState.with do |redis_shared_state| + redis_shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index f05fc565307e..5e5a90630559 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -311,9 +311,9 @@ end def expect_value_in_queues - Gitlab::Queues.with do |queues| + Gitlab::RedisQueues.with do |redis_queues| runner_queue_key = runner.send(:runner_queue_key) - expect(queues.get(runner_queue_key)) + expect(redis_queues.get(runner_queue_key)) end end end @@ -330,8 +330,8 @@ def expect_value_in_queues end it 'cleans up the queue' do - Gitlab::Queues.with do |queues| - expect(queues.get(queue_key)).to be_nil + Gitlab::RedisQueues.with do |redis_queues| + expect(redis_queues.get(queue_key)).to be_nil end end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index bf2268a25866..f2849a4d0331 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -344,7 +344,7 @@ end end - describe '#update_tracked_fields!', :gitlab_shared_state do + describe '#update_tracked_fields!', :gitlab_redis_shared_state do let(:request) { OpenStruct.new(remote_ip: "127.0.0.1") } let(:user) { create(:user) } @@ -1605,7 +1605,7 @@ def add_user(access) it { expect(user.nested_groups_projects).to eq([nested_project]) } end - describe '#refresh_authorized_projects', gitlab_shared_state: true do + describe '#refresh_authorized_projects', gitlab_redis_shared_state: true do let(:project1) { create(:empty_project) } let(:project2) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 9a4cd15c0668..b01bab351f32 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -146,7 +146,7 @@ end end - describe "POST /internal/allowed", :gitlab_shared_state do + describe "POST /internal/allowed", :gitlab_redis_shared_state do context "access granted" do before do project.team << [user, :developer] diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index cf769c333ac2..9ca6130d10be 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1187,7 +1187,7 @@ end end - context "user activities", :gitlab_shared_state do + context "user activities", :gitlab_redis_shared_state do let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 46ab4921f8d4..1129cf49b1a1 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -257,7 +257,7 @@ end end - it 'updates the user last activity', :gitlab_shared_state do + it 'updates the user last activity', :gitlab_redis_shared_state do expect(user_activity(user)).to be_nil download(path, env) do |response| diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index e120fb69a509..e366144e6527 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -98,7 +98,7 @@ def hashed_subject expect(@payload['sub']).to eq hashed_subject end - it 'includes the time of the last authentication', :gitlab_shared_state do + it 'includes the time of the last authentication', :gitlab_redis_shared_state do expect(@payload['auth_time']).to eq user.current_sign_in_at.to_i end diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb index 939d71296a36..7d161ae94477 100644 --- a/spec/services/event_create_service_spec.rb +++ b/spec/services/event_create_service_spec.rb @@ -113,7 +113,7 @@ end end - describe '#push', :gitlab_shared_state do + describe '#push', :gitlab_redis_shared_state do let(:project) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index ee912ffa32e9..c3d04edb7e6d 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -526,17 +526,17 @@ before do # Flush any raw key-value data stored by the housekeeping code. - Gitlab::Caching.with { |conn| conn.flushall } - Gitlab::Queues.with { |conn| conn.flushall } - Gitlab::SharedState.with { |conn| conn.flushall } + Gitlab::RedisCaching.with { |conn| conn.flushall } + Gitlab::RedisQueues.with { |conn| conn.flushall } + Gitlab::RedisSharedState.with { |conn| conn.flushall } allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping) end after do - Gitlab::Caching.with { |conn| conn.flushall } - Gitlab::Queues.with { |conn| conn.flushall } - Gitlab::SharedState.with { |conn| conn.flushall } + Gitlab::RedisCaching.with { |conn| conn.flushall } + Gitlab::RedisQueues.with { |conn| conn.flushall } + Gitlab::RedisSharedState.with { |conn| conn.flushall } end it 'does not perform housekeeping when not needed' do diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 3c6ada04154e..0076cc74f129 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::SharedState::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::RedisSharedState::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Gitlab::SharedState::CannotConnectError + rescue Gitlab::RedisSharedState::CannotConnectError end end diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb index 3dd819d66511..1171536a6f62 100644 --- a/spec/services/users/activity_service_spec.rb +++ b/spec/services/users/activity_service_spec.rb @@ -7,7 +7,7 @@ subject(:service) { described_class.new(user, 'type') } - describe '#execute', :gitlab_shared_state do + describe '#execute', :gitlab_redis_shared_state do context 'when last activity is nil' do before do service.execute diff --git a/spec/services/users/refresh_authorized_projects_service_spec.rb b/spec/services/users/refresh_authorized_projects_service_spec.rb index c7f56dc0f17b..db5c090836e8 100644 --- a/spec/services/users/refresh_authorized_projects_service_spec.rb +++ b/spec/services/users/refresh_authorized_projects_service_spec.rb @@ -10,7 +10,7 @@ def create_authorization(project, user, access_level = Gitlab::Access::MASTER) create!(project: project, user: user, access_level: access_level) end - describe '#execute', :gitlab_shared_state do + describe '#execute', :gitlab_redis_shared_state do it 'refreshes the authorizations using a lease' do expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain). and_return('foo') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bc535925e4f9..cdcb83c53051 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,7 @@ require './spec/simplecov_env' -require './lib/gitlab/caching' -require './lib/gitlab/queues' -require './lib/gitlab/shared_state' +require './lib/gitlab/redis_caching' +require './lib/gitlab/redis_queues' +require './lib/gitlab/redis_shared_state' SimpleCovEnv.start! ENV["RAILS_ENV"] ||= 'test' @@ -82,21 +82,21 @@ config.around(:each, :caching) do |example| caching_store = Rails.cache Rails.cache = ActiveSupport::Cache::MemoryStore.new if example.metadata[:caching] - Gitlab::Caching.with(&:flushall) + Gitlab::RedisCaching.with(&:flushall) example.run - Gitlab::Caching.with(&:flushall) + Gitlab::RedisCaching.with(&:flushall) Rails.cache = caching_store end - config.around(:each, :gitlab_shared_state) do |example| - Gitlab::SharedState.with(&:flushall) + config.around(:each, :gitlab_redis_shared_state) do |example| + Gitlab::RedisSharedState.with(&:flushall) Sidekiq.redis(&:flushall) example.run - Gitlab::SharedState.with(&:flushall) + Gitlab::RedisSharedState.with(&:flushall) Sidekiq.redis(&:flushall) end end diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 7a69ac34688a..3a72b2c41a71 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,9 +1,9 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::Caching.with(&:flushall) - Gitlab::Queues.with(&:flushall) - Gitlab::SharedState.with(&:flushall) + Gitlab::RedisCaching.with(&:flushall) + Gitlab::RedisQueues.with(&:flushall) + Gitlab::RedisSharedState.with(&:flushall) end before do diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb index b8e8debe62f7..dace57f3c2a3 100644 --- a/spec/workers/schedule_update_user_activity_worker_spec.rb +++ b/spec/workers/schedule_update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ScheduleUpdateUserActivityWorker, :gitlab_shared_state do +describe ScheduleUpdateUserActivityWorker, :gitlab_redis_shared_state do let(:now) { Time.now } before do diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb index 44ecb437e6af..4393bdade37a 100644 --- a/spec/workers/update_user_activity_worker_spec.rb +++ b/spec/workers/update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe UpdateUserActivityWorker, :gitlab_shared_state do +describe UpdateUserActivityWorker, :gitlab_redis_shared_state do let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) } let(:user_active_yesterday_1) { create(:user) } let(:user_active_yesterday_2) { create(:user) } -- GitLab From f7f29b8f996129e11299f2a52dbd23f0551813e3 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sun, 28 May 2017 17:41:23 +0800 Subject: [PATCH 05/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- lib/tasks/cache.rake | 2 +- spec/controllers/sessions_controller_spec.rb | 2 +- spec/features/dashboard/projects_spec.rb | 2 +- spec/features/groups/members/sorting_spec.rb | 4 ++-- spec/features/login_spec.rb | 4 ++-- spec/features/projects/members/sorting_spec.rb | 4 ++-- spec/helpers/issuables_helper_spec.rb | 2 +- spec/helpers/projects_helper_spec.rb | 2 +- spec/lib/gitlab/auth/unique_ips_limiter_spec.rb | 2 +- .../cache/ci/project_pipeline_status_spec.rb | 10 +++++----- spec/lib/gitlab/exclusive_lease_spec.rb | 2 +- spec/lib/gitlab/sidekiq_status_spec.rb | 12 ++++++------ spec/lib/gitlab/user_activities_spec.rb | 2 +- .../migrate_process_commit_worker_jobs_spec.rb | 4 ++-- ...er_activities_to_users_last_activity_on_spec.rb | 2 +- spec/models/user_spec.rb | 4 ++-- spec/requests/api/internal_spec.rb | 2 +- spec/requests/api/users_spec.rb | 2 +- spec/requests/git_http_spec.rb | 2 +- spec/requests/openid_connect_spec.rb | 2 +- spec/services/event_create_service_spec.rb | 2 +- spec/services/todo_service_spec.rb | 2 +- spec/services/users/activity_service_spec.rb | 2 +- .../refresh_authorized_projects_service_spec.rb | 2 +- spec/spec_helper.rb | 14 ++++++++++---- .../schedule_update_user_activity_worker_spec.rb | 2 +- spec/workers/update_user_activity_worker_spec.rb | 2 +- 27 files changed, 50 insertions(+), 44 deletions(-) diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 0560f37937be..5f2b6a2cdd7c 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -22,7 +22,7 @@ namespace :cache do end end - task all: [:redis] + task all: [:gitlab_kv_cache] end task clear: 'cache:clear:gitlab_kv_cache' diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index d7443a5943d0..43c4c11bd9de 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -16,7 +16,7 @@ end end - context 'when using valid password', :gitlab_redis_shared_state do + context 'when using valid password', :clean_gitlab_redis_shared_state do include UserActivitiesHelpers let(:user) { create(:user) } diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb index 4a853ffc75ab..3a739deb5660 100644 --- a/spec/features/dashboard/projects_spec.rb +++ b/spec/features/dashboard/projects_spec.rb @@ -14,7 +14,7 @@ expect(page).to have_content('awesome stuff') end - describe "with a pipeline", gitlab_redis_shared_state: true do + describe "with a pipeline", clean_gitlab_redis_shared_state: true do let!(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) } before do diff --git a/spec/features/groups/members/sorting_spec.rb b/spec/features/groups/members/sorting_spec.rb index 8b2735372fc6..cbdfe2c750e5 100644 --- a/spec/features/groups/members/sorting_spec.rb +++ b/spec/features/groups/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :gitlab_redis_shared_state do + scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(owner.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :gitlab_redis_shared_state do + scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb index 8593ef922050..f8b8fcdf5904 100644 --- a/spec/features/login_spec.rb +++ b/spec/features/login_spec.rb @@ -41,7 +41,7 @@ expect(page).to have_content('Your account has been blocked.') end - it 'does not update Devise trackable attributes', :gitlab_redis_shared_state do + it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do user = create(:user, :blocked) expect { login_with(user) }.not_to change { user.reload.sign_in_count } @@ -55,7 +55,7 @@ expect(page).to have_content('Invalid Login or password.') end - it 'does not update Devise trackable attributes', :gitlab_redis_shared_state do + it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do expect { login_with(User.ghost) }.not_to change { User.ghost.reload.sign_in_count } end end diff --git a/spec/features/projects/members/sorting_spec.rb b/spec/features/projects/members/sorting_spec.rb index 714d9cf4d63a..e596ee17b5bd 100644 --- a/spec/features/projects/members/sorting_spec.rb +++ b/spec/features/projects/members/sorting_spec.rb @@ -68,7 +68,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :gitlab_redis_shared_state do + scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(master.name) @@ -76,7 +76,7 @@ expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :gitlab_redis_shared_state do + scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/helpers/issuables_helper_spec.rb b/spec/helpers/issuables_helper_spec.rb index c1ecb46aece8..9f4a447d988a 100644 --- a/spec/helpers/issuables_helper_spec.rb +++ b/spec/helpers/issuables_helper_spec.rb @@ -60,7 +60,7 @@ end end - describe 'counter caching based on issuable type and params', :caching do + describe 'counter caching based on issuable type and params', :use_clean_rails_memory_store_caching do let(:params) do { scope: 'created-by-me', diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index b424ccc82d9e..79efd19cd17f 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -63,7 +63,7 @@ end end - describe "#project_list_cache_key", gitlab_redis_shared_state: true do + describe "#project_list_cache_key", clean_gitlab_redis_shared_state: true do let(:project) { create(:project) } it "includes the route" do diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb index 6af575f28cea..8d500469cd40 100644 --- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb +++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Auth::UniqueIpsLimiter, :gitlab_redis_shared_state, lib: true do +describe Gitlab::Auth::UniqueIpsLimiter, :clean_gitlab_redis_shared_state, lib: true do include_context 'unique ips sign in limit' let(:user) { create(:user) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index a9d9db4adead..afb777525ecd 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus, :caching do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_caching do let(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } let(:cache_key) { "projects/#{project.id}/pipeline_status" } @@ -183,7 +183,7 @@ end end - describe "#load_from_project", :caching do + describe "#load_from_project", :clean_gitlab_redis_caching do let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } it 'reads the status from the pipeline for the commit' do @@ -203,7 +203,7 @@ end end - describe "#store_in_cache", :caching do + describe "#store_in_cache", :clean_gitlab_redis_caching do it "sets the object in caching" do pipeline_status.sha = '123456' pipeline_status.status = 'failed' @@ -216,7 +216,7 @@ end end - describe '#store_in_cache_if_needed', :caching do + describe '#store_in_cache_if_needed', :clean_gitlab_redis_caching do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) pipeline_status = described_class.load_for_project(project) @@ -263,7 +263,7 @@ end end - describe "with a status in caching", :caching do + describe "with a status in caching", :clean_gitlab_redis_caching do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:ref) { 'master' } diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb index 5bfd4b25d3a9..d31fd2670ee3 100644 --- a/spec/lib/gitlab/exclusive_lease_spec.rb +++ b/spec/lib/gitlab/exclusive_lease_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::ExclusiveLease, type: :gitlab_redis_shared_state do +describe Gitlab::ExclusiveLease, type: :clean_gitlab_redis_shared_state do let(:unique_key) { SecureRandom.hex(10) } describe '#try_obtain' do diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index b0efba8e0ade..c2e77ef6b6c4 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::SidekiqStatus do - describe '.set', :gitlab_redis_shared_state do + describe '.set', :clean_gitlab_redis_shared_state do it 'stores the job ID' do described_class.set('123') @@ -14,7 +14,7 @@ end end - describe '.unset', :gitlab_redis_shared_state do + describe '.unset', :clean_gitlab_redis_shared_state do it 'removes the job ID' do described_class.set('123') described_class.unset('123') @@ -27,7 +27,7 @@ end end - describe '.all_completed?', :gitlab_redis_shared_state do + describe '.all_completed?', :clean_gitlab_redis_shared_state do it 'returns true if all jobs have been completed' do expect(described_class.all_completed?(%w(123))).to eq(true) end @@ -39,7 +39,7 @@ end end - describe '.num_running', :gitlab_redis_shared_state do + describe '.num_running', :clean_gitlab_redis_shared_state do it 'returns 0 if all jobs have been completed' do expect(described_class.num_running(%w(123))).to eq(0) end @@ -52,7 +52,7 @@ end end - describe '.num_completed', :gitlab_redis_shared_state do + describe '.num_completed', :clean_gitlab_redis_shared_state do it 'returns 1 if all jobs have been completed' do expect(described_class.num_completed(%w(123))).to eq(1) end @@ -74,7 +74,7 @@ end end - describe 'completed', :gitlab_redis_shared_state do + describe 'completed', :clean_gitlab_redis_shared_state do it 'returns the completed job' do expect(described_class.completed_jids(%w(123))).to eq(['123']) end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index af774b7daac4..b8190deae995 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::UserActivities, :gitlab_redis_shared_state, lib: true do +describe Gitlab::UserActivities, :clean_gitlab_redis_shared_state, lib: true do let(:now) { Time.now } describe '.record' do diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb index 131214cc7008..76e5f5b3fff6 100644 --- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb +++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb @@ -54,7 +54,7 @@ end end - describe '#up', :gitlab_redis_shared_state do + describe '#up', :clean_gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count @@ -172,7 +172,7 @@ def pop_job end end - describe '#down', :gitlab_redis_shared_state do + describe '#down', :clean_gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index c3c1ab41259e..b5cebd55696e 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') -describe MigrateUserActivitiesToUsersLastActivityOn, :gitlab_redis_shared_state do +describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state do let(:migration) { described_class.new } let!(:user_active_1) { create(:user) } let!(:user_active_2) { create(:user) } diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index f2849a4d0331..91b8818e43b1 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -344,7 +344,7 @@ end end - describe '#update_tracked_fields!', :gitlab_redis_shared_state do + describe '#update_tracked_fields!', :clean_gitlab_redis_shared_state do let(:request) { OpenStruct.new(remote_ip: "127.0.0.1") } let(:user) { create(:user) } @@ -1605,7 +1605,7 @@ def add_user(access) it { expect(user.nested_groups_projects).to eq([nested_project]) } end - describe '#refresh_authorized_projects', gitlab_redis_shared_state: true do + describe '#refresh_authorized_projects', clean_gitlab_redis_shared_state: true do let(:project1) { create(:empty_project) } let(:project2) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index b01bab351f32..1ba6e85c40e1 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -146,7 +146,7 @@ end end - describe "POST /internal/allowed", :gitlab_redis_shared_state do + describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do context "access granted" do before do project.team << [user, :developer] diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index 9ca6130d10be..68f65775eda9 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1187,7 +1187,7 @@ end end - context "user activities", :gitlab_redis_shared_state do + context "user activities", :clean_gitlab_redis_shared_state do let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 1129cf49b1a1..eadcc728ba98 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -257,7 +257,7 @@ end end - it 'updates the user last activity', :gitlab_redis_shared_state do + it 'updates the user last activity', :clean_gitlab_redis_shared_state do expect(user_activity(user)).to be_nil download(path, env) do |response| diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index e366144e6527..4c3371888c94 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -98,7 +98,7 @@ def hashed_subject expect(@payload['sub']).to eq hashed_subject end - it 'includes the time of the last authentication', :gitlab_redis_shared_state do + it 'includes the time of the last authentication', :clean_gitlab_redis_shared_state do expect(@payload['auth_time']).to eq user.current_sign_in_at.to_i end diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb index 7d161ae94477..8d067c194cc0 100644 --- a/spec/services/event_create_service_spec.rb +++ b/spec/services/event_create_service_spec.rb @@ -113,7 +113,7 @@ end end - describe '#push', :gitlab_redis_shared_state do + describe '#push', :clean_gitlab_redis_shared_state do let(:project) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/services/todo_service_spec.rb b/spec/services/todo_service_spec.rb index 175a42a32d9b..de41cbab14cb 100644 --- a/spec/services/todo_service_spec.rb +++ b/spec/services/todo_service_spec.rb @@ -908,7 +908,7 @@ end end - it 'caches the number of todos of a user', :caching do + it 'caches the number of todos of a user', :use_clean_rails_memory_store_caching do create(:todo, :mentioned, user: john_doe, target: issue, project: project) todo = create(:todo, :mentioned, user: john_doe, target: issue, project: project) TodoService.new.mark_todos_as_done([todo], john_doe) diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb index 1171536a6f62..c7bc8738855e 100644 --- a/spec/services/users/activity_service_spec.rb +++ b/spec/services/users/activity_service_spec.rb @@ -7,7 +7,7 @@ subject(:service) { described_class.new(user, 'type') } - describe '#execute', :gitlab_redis_shared_state do + describe '#execute', :clean_gitlab_redis_shared_state do context 'when last activity is nil' do before do service.execute diff --git a/spec/services/users/refresh_authorized_projects_service_spec.rb b/spec/services/users/refresh_authorized_projects_service_spec.rb index db5c090836e8..e971ff1230c2 100644 --- a/spec/services/users/refresh_authorized_projects_service_spec.rb +++ b/spec/services/users/refresh_authorized_projects_service_spec.rb @@ -10,7 +10,7 @@ def create_authorization(project, user, access_level = Gitlab::Access::MASTER) create!(project: project, user: user, access_level: access_level) end - describe '#execute', :gitlab_redis_shared_state do + describe '#execute', :clean_gitlab_redis_shared_state do it 'refreshes the authorizations using a lease' do expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain). and_return('foo') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cdcb83c53051..8b56d667fce7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -79,18 +79,24 @@ end end - config.around(:each, :caching) do |example| + config.around(:each, :use_clean_rails_memory_store_caching) do |example| caching_store = Rails.cache - Rails.cache = ActiveSupport::Cache::MemoryStore.new if example.metadata[:caching] + Rails.cache = ActiveSupport::Cache::MemoryStore.new + + example.run + + Rails.cache = caching_store + end + + config.around(:each, :clean_gitlab_redis_caching) do |example| Gitlab::RedisCaching.with(&:flushall) example.run Gitlab::RedisCaching.with(&:flushall) - Rails.cache = caching_store end - config.around(:each, :gitlab_redis_shared_state) do |example| + config.around(:each, :clean_gitlab_redis_shared_state) do |example| Gitlab::RedisSharedState.with(&:flushall) Sidekiq.redis(&:flushall) diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb index dace57f3c2a3..32c59381b01f 100644 --- a/spec/workers/schedule_update_user_activity_worker_spec.rb +++ b/spec/workers/schedule_update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ScheduleUpdateUserActivityWorker, :gitlab_redis_shared_state do +describe ScheduleUpdateUserActivityWorker, :clean_gitlab_redis_shared_state do let(:now) { Time.now } before do diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb index 4393bdade37a..268ca1d81f28 100644 --- a/spec/workers/update_user_activity_worker_spec.rb +++ b/spec/workers/update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe UpdateUserActivityWorker, :gitlab_redis_shared_state do +describe UpdateUserActivityWorker, :clean_gitlab_redis_shared_state do let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) } let(:user_active_yesterday_1) { create(:user) } let(:user_active_yesterday_2) { create(:user) } -- GitLab From 19f3061a222350317ad3a2840cbf865b8177da50 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sun, 28 May 2017 18:57:18 +0800 Subject: [PATCH 06/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- .../dashboard/issuables_counter_spec.rb | 2 +- spec/models/concerns/reactive_caching_spec.rb | 2 +- .../project_services/bamboo_service_spec.rb | 2 +- .../project_services/buildkite_service_spec.rb | 2 +- .../project_services/drone_ci_service_spec.rb | 2 +- .../kubernetes_service_spec.rb | 2 +- .../prometheus_service_spec.rb | 2 +- .../project_services/teamcity_service_spec.rb | 2 +- spec/models/project_spec.rb | 2 +- spec/models/repository_spec.rb | 18 +++++++++--------- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/spec/features/dashboard/issuables_counter_spec.rb b/spec/features/dashboard/issuables_counter_spec.rb index 354267dbee7f..60dd71e88e84 100644 --- a/spec/features/dashboard/issuables_counter_spec.rb +++ b/spec/features/dashboard/issuables_counter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe 'Navigation bar counter', feature: true, caching: true do +describe 'Navigation bar counter', :use_clean_rails_memory_store_caching, feature: true do let(:user) { create(:user) } let(:project) { create(:empty_project, namespace: user.namespace) } let(:issue) { create(:issue, project: project) } diff --git a/spec/models/concerns/reactive_caching_spec.rb b/spec/models/concerns/reactive_caching_spec.rb index a0765a264cfc..7ae489ca1a1a 100644 --- a/spec/models/concerns/reactive_caching_spec.rb +++ b/spec/models/concerns/reactive_caching_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ReactiveCaching, caching: true do +describe ReactiveCaching, :use_clean_rails_memory_store_caching do include ReactiveCachingHelpers class CacheTest diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb index 4014d6129eee..78cd497ea760 100644 --- a/spec/models/project_services/bamboo_service_spec.rb +++ b/spec/models/project_services/bamboo_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe BambooService, models: true, caching: true do +describe BambooService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:bamboo_url) { 'http://gitlab.com/bamboo' } diff --git a/spec/models/project_services/buildkite_service_spec.rb b/spec/models/project_services/buildkite_service_spec.rb index 05b602d81065..433dff15d7d4 100644 --- a/spec/models/project_services/buildkite_service_spec.rb +++ b/spec/models/project_services/buildkite_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe BuildkiteService, models: true, caching: true do +describe BuildkiteService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:project) { create(:empty_project) } diff --git a/spec/models/project_services/drone_ci_service_spec.rb b/spec/models/project_services/drone_ci_service_spec.rb index 044737c6026d..0dd2d7e7783b 100644 --- a/spec/models/project_services/drone_ci_service_spec.rb +++ b/spec/models/project_services/drone_ci_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe DroneCiService, models: true, caching: true do +describe DroneCiService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers describe 'associations' do diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index c1c2f2a72198..5920dcbe38f4 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe KubernetesService, models: true, caching: true do +describe KubernetesService, :use_clean_rails_memory_store_caching, models: true do include KubernetesHelpers include ReactiveCachingHelpers diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index 1f9d3c07b513..80820d450f24 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe PrometheusService, models: true, caching: true do +describe PrometheusService, :use_clean_rails_memory_store_caching, models: true do include PrometheusHelpers include ReactiveCachingHelpers diff --git a/spec/models/project_services/teamcity_service_spec.rb b/spec/models/project_services/teamcity_service_spec.rb index 77b18e1c7d0f..37b7fc748601 100644 --- a/spec/models/project_services/teamcity_service_spec.rb +++ b/spec/models/project_services/teamcity_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe TeamcityService, models: true, caching: true do +describe TeamcityService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:teamcity_url) { 'http://gitlab.com/teamcity' } diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index f2b4e9070b41..7c3406d46f5d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -904,7 +904,7 @@ def create_pipeline end end - describe '.cached_count', caching: true do + describe '.cached_count', :use_clean_rails_memory_store_caching do let(:group) { create(:group, :public) } let!(:project1) { create(:empty_project, :public, group: group) } let!(:project2) { create(:empty_project, :public, group: group) } diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 718b7d5e86b8..28bd4c2b043e 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -550,7 +550,7 @@ end end - describe "#changelog", caching: true do + describe "#changelog", :use_clean_rails_memory_store_caching do it 'accepts changelog' do expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')]) @@ -582,7 +582,7 @@ end end - describe "#license_blob", caching: true do + describe "#license_blob", :use_clean_rails_memory_store_caching do before do repository.delete_file( user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master') @@ -627,7 +627,7 @@ end end - describe '#license_key', caching: true do + describe '#license_key', :use_clean_rails_memory_store_caching do before do repository.delete_file(user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master') @@ -692,7 +692,7 @@ end end - describe "#gitlab_ci_yml", caching: true do + describe "#gitlab_ci_yml", :use_clean_rails_memory_store_caching do it 'returns valid file' do files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')] expect(repository.tree).to receive(:blobs).and_return(files) @@ -1605,7 +1605,7 @@ def merge(repository, user, merge_request, options = {}) end end - describe '#contribution_guide', caching: true do + describe '#contribution_guide', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head). with(:contributing). @@ -1619,7 +1619,7 @@ def merge(repository, user, merge_request, options = {}) end end - describe '#gitignore', caching: true do + describe '#gitignore', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head). with(:gitignore). @@ -1632,7 +1632,7 @@ def merge(repository, user, merge_request, options = {}) end end - describe '#koding_yml', caching: true do + describe '#koding_yml', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head). with(:koding). @@ -1645,7 +1645,7 @@ def merge(repository, user, merge_request, options = {}) end end - describe '#readme', caching: true do + describe '#readme', :use_clean_rails_memory_store_caching do context 'with a non-existing repository' do it 'returns nil' do allow(repository).to receive(:tree).with(:head).and_return(nil) @@ -1816,7 +1816,7 @@ def merge(repository, user, merge_request, options = {}) end end - describe '#cache_method_output', caching: true do + describe '#cache_method_output', :use_clean_rails_memory_store_caching do context 'with a non-existing repository' do let(:value) do repository.cache_method_output(:cats, fallback: 10) do -- GitLab From f4c5ee44b4bba8fa6640ce09a19940a6c8f774b0 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 08:04:53 +0800 Subject: [PATCH 07/62] create new submodule --- app/models/ci/runner.rb | 2 +- app/models/project.rb | 6 +- config/application.rb | 10 +- config/initializers/7_redis.rb | 6 +- config/initializers/session_store.rb | 4 +- config/initializers/sidekiq.rb | 6 +- config/mail_room.yml | 4 +- ...er_activities_to_users_last_activity_on.rb | 4 +- lib/gitlab/auth/unique_ips_limiter.rb | 2 +- .../cache/ci/project_pipeline_status.rb | 10 +- lib/gitlab/chat_name_token.rb | 6 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/etag_caching/store.rb | 4 +- lib/gitlab/exclusive_lease.rb | 6 +- .../health_checks/redis_caching_check.rb | 2 +- .../health_checks/redis_queues_check.rb | 2 +- .../health_checks/redis_shared_state_check.rb | 2 +- lib/gitlab/lfs_token.rb | 2 +- lib/gitlab/mail_room.rb | 4 +- lib/gitlab/redis/caching.rb | 24 +++++ lib/gitlab/redis/queues.rb | 25 +++++ lib/gitlab/redis/shared_state.rb | 24 +++++ lib/gitlab/redis/wrapper.rb | 99 +++++++++++++++++++ lib/gitlab/redis_caching.rb | 22 ----- lib/gitlab/redis_queues.rb | 23 ----- lib/gitlab/redis_shared_state.rb | 22 ----- lib/gitlab/redis_wrapper.rb | 97 ------------------ lib/gitlab/user_activities.rb | 6 +- lib/gitlab/workhorse.rb | 2 +- lib/tasks/cache.rake | 6 +- spec/config/mail_room_spec.rb | 4 +- .../cache/ci/project_pipeline_status_spec.rb | 22 ++--- spec/lib/gitlab/current_settings_spec.rb | 2 +- spec/lib/gitlab/redis_caching_spec.rb | 4 +- spec/lib/gitlab/redis_queues_spec.rb | 4 +- spec/lib/gitlab/redis_shared_state_spec.rb | 4 +- spec/lib/gitlab/user_activities_spec.rb | 20 ++-- ...tivities_to_users_last_activity_on_spec.rb | 2 +- spec/models/ci/runner_spec.rb | 4 +- spec/services/git_push_service_spec.rb | 12 +-- .../services/projects/destroy_service_spec.rb | 4 +- spec/spec_helper.rb | 14 +-- .../unique_ip_check_shared_examples.rb | 6 +- 43 files changed, 272 insertions(+), 264 deletions(-) create mode 100644 lib/gitlab/redis/caching.rb create mode 100644 lib/gitlab/redis/queues.rb create mode 100644 lib/gitlab/redis/shared_state.rb create mode 100644 lib/gitlab/redis/wrapper.rb delete mode 100644 lib/gitlab/redis_caching.rb delete mode 100644 lib/gitlab/redis_queues.rb delete mode 100644 lib/gitlab/redis_shared_state.rb delete mode 100644 lib/gitlab/redis_wrapper.rb diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 4907f65e2d5a..d2be57e36ff0 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,7 +145,7 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| redis_queues.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index eb16af80fbf7..16bdde93c096 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1280,15 +1280,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) diff --git a/config/application.rb b/config/application.rb index de565d2edacd..af78c06e3219 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,9 +6,9 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis_caching') - require_dependency Rails.root.join('lib/gitlab/redis_queues') - require_dependency Rails.root.join('lib/gitlab/redis_shared_state') + require_dependency Rails.root.join('lib/gitlab/redis/caching') + require_dependency Rails.root.join('lib/gitlab/redis/queues') + require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -141,8 +141,8 @@ class Application < Rails::Application end # Use caching across all environments - caching_config_hash = Gitlab::RedisCaching.params - caching_config_hash[:namespace] = Gitlab::RedisCaching::CACHE_NAMESPACE + caching_config_hash = Gitlab::Redis::Caching.params + caching_config_hash[:namespace] = Gitlab::Redis::Caching::CACHE_NAMESPACE caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 797361b5968f..7ad600aa26ae 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::RedisCaching.with { nil } -Gitlab::RedisQueues.with { nil } -Gitlab::RedisSharedState.with { nil } +Gitlab::Redis::Caching.with { nil } +Gitlab::Redis::Queues.with { nil } +Gitlab::Redis::SharedState.with { nil } diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 22256d0ef13b..87768095859b 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,8 +13,8 @@ if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - sessions_config = Gitlab::RedisSharedState.params - sessions_config[:namespace] = Gitlab::RedisSharedState::SESSION_NAMESPACE + sessions_config = Gitlab::Redis::SharedState.params + sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 27e18fa960f7..bf99b248bbcb 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,6 +1,6 @@ # Custom Queues configuration -queues_config_hash = Gitlab::RedisQueues.params -queues_config_hash[:namespace] = Gitlab::RedisQueues::SIDEKIQ_NAMESPACE +queues_config_hash = Gitlab::Redis::Queues.params +queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } @@ -74,5 +74,5 @@ end end end -rescue Gitlab::RedisQueues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue Gitlab::Redis::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/config/mail_room.yml b/config/mail_room.yml index d191400e8105..c3a5be8d38c5 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::RedisQueues::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::RedisQueues::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb index f8e0895a53f8..e30b8788234c 100644 --- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -56,7 +56,7 @@ def users_table end def activities(from, to, page: 1) - Gitlab::RedisSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, with_scores: true, limit: limit(page)) @@ -64,7 +64,7 @@ def activities(from, to, page: 1) end def activities_count(from, to) - Gitlab::RedisSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) end end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 7b82521cdd4e..6fff4309e893 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| unique_ips_count = nil redis_shared_state.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 414e994c8f24..492e1254f164 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::RedisCaching.with do |redis_caching| + result = Gitlab::Redis::Caching.with do |redis_caching| redis_caching.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| self.sha, self.status, self.ref = redis_caching.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.del(cache_key) end end @@ -120,7 +120,7 @@ def delete_from_cache def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index d4a858dbdd4c..bf8d442140fc 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,14 +12,14 @@ def initialize(token = new_token) end def get - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| data = redis_shared_state.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| params = params.to_json redis_shared_state.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token @@ -27,7 +27,7 @@ def store!(params) end def delete - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.del(redis_shared_state_key) end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9d07afc29a19..88fd408ef475 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::RedisCaching::BaseError, ::Errno::ENOENT + rescue ::Gitlab::Redis::Caching::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 587580102ed9..7096be42beab 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -5,13 +5,13 @@ class Store SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a2c6a2933cbd..19f3b7b50b74 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,7 +18,7 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end @@ -37,14 +37,14 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.exists(@redis_shared_state_key) end end diff --git a/lib/gitlab/health_checks/redis_caching_check.rb b/lib/gitlab/health_checks/redis_caching_check.rb index 5a56cc37246f..767c5adddf1c 100644 --- a/lib/gitlab/health_checks/redis_caching_check.rb +++ b/lib/gitlab/health_checks/redis_caching_check.rb @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::RedisCaching.with(&:ping) + Gitlab::Redis::Caching.with(&:ping) end end end diff --git a/lib/gitlab/health_checks/redis_queues_check.rb b/lib/gitlab/health_checks/redis_queues_check.rb index 2b0d270fb587..b6ce6fe82e81 100644 --- a/lib/gitlab/health_checks/redis_queues_check.rb +++ b/lib/gitlab/health_checks/redis_queues_check.rb @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::RedisQueues.with(&:ping) + Gitlab::Redis::Queues.with(&:ping) end end end diff --git a/lib/gitlab/health_checks/redis_shared_state_check.rb b/lib/gitlab/health_checks/redis_shared_state_check.rb index 68a5c4fce593..a478576ba8d4 100644 --- a/lib/gitlab/health_checks/redis_shared_state_check.rb +++ b/lib/gitlab/health_checks/redis_shared_state_check.rb @@ -16,7 +16,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::RedisSharedState.with(&:ping) + Gitlab::Redis::SharedState.with(&:ping) end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 4a837401592c..043bfe18545b 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,7 +18,7 @@ def initialize(actor) end def token - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| token = redis_shared_state.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) redis_shared_state.set(redis_shared_state_key, token, ex: EXPIRY_TIME) diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 88eb0afa91ab..a2b363e54085 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis_queues' unless defined?(Gitlab::RedisQueues) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,7 +34,7 @@ def fetch_config config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis_queues = Gitlab::RedisQueues.new(rails_env) + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) config[:redis_url] = gitlab_redis_queues.url # @@ TODO this is too-much-information on topology in the client of an object diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb new file mode 100644 index 000000000000..e4214fe761d7 --- /dev/null +++ b/lib/gitlab/redis/caching.rb @@ -0,0 +1,24 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class Caching < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || File.expand_path('../../../config/redis.caching.yml', __dir__) + end + + def default_url + DEFAULT_CACHING_URL + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 000000000000..d7e8d9378f97 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,25 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || File.expand_path('../../../config/redis.queues.yml', __dir__) + end + + def default_url + DEFAULT_QUEUES_URL + end + end + end +end \ No newline at end of file diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 000000000000..23e1564fb65f --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,24 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + + class << self + delegate :_raw_config, :fetch_config, to: :new + end + + def config_file + ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../../config/redis.shared_state.yml', __dir__) + end + + def default_url + DEFAULT_SHARED_STATE_URL + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 000000000000..c069ac7da4a8 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,99 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'rails' +require 'redis' +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + BaseError = ::Redis::BaseError.freeze + CannotConnectError = ::Redis::CannotConnectError.freeze + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(config_file)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: default_url } + end + end + + def fetch_config + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false + end + end + end +end \ No newline at end of file diff --git a/lib/gitlab/redis_caching.rb b/lib/gitlab/redis_caching.rb deleted file mode 100644 index 9636c1219948..000000000000 --- a/lib/gitlab/redis_caching.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisCaching < ::Gitlab::RedisWrapper - CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || File.expand_path('../../config/redis.caching.yml', __dir__) - end - - def default_url - DEFAULT_CACHING_URL - end - end -end diff --git a/lib/gitlab/redis_queues.rb b/lib/gitlab/redis_queues.rb deleted file mode 100644 index 3cdd9cde17cf..000000000000 --- a/lib/gitlab/redis_queues.rb +++ /dev/null @@ -1,23 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisQueues < ::Gitlab::RedisWrapper - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || File.expand_path('../../config/redis.queues.yml', __dir__) - end - - def default_url - DEFAULT_QUEUES_URL - end - end -end diff --git a/lib/gitlab/redis_shared_state.rb b/lib/gitlab/redis_shared_state.rb deleted file mode 100644 index cba6c30d5024..000000000000 --- a/lib/gitlab/redis_shared_state.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisSharedState < ::Gitlab::RedisWrapper - SESSION_NAMESPACE = 'session:gitlab'.freeze - DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../config/redis.shared_state.yml', __dir__) - end - - def default_url - DEFAULT_SHARED_STATE_URL - end - end -end diff --git a/lib/gitlab/redis_wrapper.rb b/lib/gitlab/redis_wrapper.rb deleted file mode 100644 index 63d247c782d5..000000000000 --- a/lib/gitlab/redis_wrapper.rb +++ /dev/null @@ -1,97 +0,0 @@ -# This file should only be used by sub-classes, not directly by any clients of the sub-classes -# please require all dependencies below: -require 'rails' -require 'redis' -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class RedisWrapper - BaseError = ::Redis::BaseError.freeze - CannotConnectError = ::Redis::CannotConnectError.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - # heuristic constant 5 should be a config setting somewhere -- related to CPU count? - size = 5 - if Sidekiq.server? - # the pool will be used in a multi-threaded context - size += Sidekiq.options[:concurrency] - end - size - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: default_url } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index be5c8a707b8b..cec80becfb95 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 2a30bd6a9e70..80bd9ca757fd 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,7 +174,7 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| result = redis_queues.set(key, value, ex: expire, nx: !overwrite) if result redis_queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 5f2b6a2cdd7c..6416b45d5ef7 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,17 +1,17 @@ namespace :cache do namespace :clear do - # @@ TODO - push implementation into Gitlab::RedisCaching + # @@ TODO - push implementation into Gitlab::Redis::Caching KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear key/value cache" task gitlab_kv_cache: :environment do - Gitlab::RedisCaching.with do |kv_cache| + Gitlab::Redis::Caching.with do |kv_cache| cursor = KV_CACHE_SCAN_START_STOP loop do cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::RedisCaching::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Caching::CACHE_NAMESPACE}*", count: KV_CACHE_CLEAR_BATCH_SIZE ) diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index e96a9d0e43b2..a31e44fa9285 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -41,7 +41,7 @@ let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - let(:gitlab_redis_queues) { Gitlab::RedisQueues.new(Rails.env) } + let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) } it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) @@ -74,7 +74,7 @@ end def clear_queues_raw_config - Gitlab::RedisQueues.remove_instance_variable(:@_raw_config) + Gitlab::Redis::Queues.remove_instance_variable(:@_raw_config) rescue NameError # raised if @_raw_config was not set; ignore end diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index afb777525ecd..2158530d59a2 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -50,7 +50,7 @@ it 'only connects to redis_caching twice' do # Once to load, once to store in the cache - expect(Gitlab::RedisCaching).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Redis::Caching).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -60,7 +60,7 @@ describe 'when a status was cached in redis_caching' do before do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -77,7 +77,7 @@ end it 'only connects to redis_caching once' do - expect(Gitlab::RedisCaching).to receive(:with).exactly(1).and_call_original + expect(Gitlab::Redis::Caching).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -95,7 +95,7 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -209,7 +209,7 @@ pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') @@ -222,7 +222,7 @@ pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil @@ -236,7 +236,7 @@ ) other_status.store_in_cache_if_needed - sha, status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,7 +244,7 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -269,7 +269,7 @@ let(:ref) { 'master' } before do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -295,7 +295,7 @@ it 'deletes values from redis_caching' do pipeline_status.delete_from_cache - key_exists = Gitlab::RedisCaching.with { |redis_caching| redis_caching.exists(cache_key) } + key_exists = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index bd1efdf20a3e..a77a24de5aef 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::RedisCaching::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Caching::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/lib/gitlab/redis_caching_spec.rb b/spec/lib/gitlab/redis_caching_spec.rb index f373716a0b39..a62e2c80fff6 100644 --- a/spec/lib/gitlab/redis_caching_spec.rb +++ b/spec/lib/gitlab/redis_caching_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisCaching do +describe Gitlab::Redis::Caching do include StubENV let(:config) { 'config/redis.caching.yml' } @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisCaching::DEFAULT_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_CACHING_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis_queues_spec.rb b/spec/lib/gitlab/redis_queues_spec.rb index b498f48cd236..b38157b0d31a 100644 --- a/spec/lib/gitlab/redis_queues_spec.rb +++ b/spec/lib/gitlab/redis_queues_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisQueues do +describe Gitlab::Redis::Queues do include StubENV let(:config) { 'config/redis.queues.yml' } @@ -180,7 +180,7 @@ it 'returns default queues url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisQueues::DEFAULT_QUEUES_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Queues::DEFAULT_QUEUES_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis_shared_state_spec.rb b/spec/lib/gitlab/redis_shared_state_spec.rb index 449691980d46..67ec76ad7052 100644 --- a/spec/lib/gitlab/redis_shared_state_spec.rb +++ b/spec/lib/gitlab/redis_shared_state_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisSharedState do +describe Gitlab::Redis::SharedState do include StubENV let(:config) { 'config/redis.shared_state.yml' } @@ -180,7 +180,7 @@ it 'returns default redis_shared_state url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisSharedState::DEFAULT_SHARED_STATE_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::SharedState::DEFAULT_SHARED_STATE_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index b8190deae995..9a9011044590 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -11,7 +11,7 @@ described_class.record(42) end - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end @@ -21,7 +21,7 @@ it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end @@ -34,13 +34,13 @@ it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -48,13 +48,13 @@ context 'and key does not exist' do it 'removes the pair from SharedState' do - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -67,13 +67,13 @@ described_class.record(41, now) described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -83,13 +83,13 @@ it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index b5cebd55696e..d30d7fd92ddb 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -9,7 +9,7 @@ let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 5e5a90630559..678c02c851fa 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -311,7 +311,7 @@ end def expect_value_in_queues - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| runner_queue_key = runner.send(:runner_queue_key) expect(redis_queues.get(runner_queue_key)) end @@ -330,7 +330,7 @@ def expect_value_in_queues end it 'cleans up the queue' do - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| expect(redis_queues.get(queue_key)).to be_nil end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index c3d04edb7e6d..c12e276c9a70 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -526,17 +526,17 @@ before do # Flush any raw key-value data stored by the housekeeping code. - Gitlab::RedisCaching.with { |conn| conn.flushall } - Gitlab::RedisQueues.with { |conn| conn.flushall } - Gitlab::RedisSharedState.with { |conn| conn.flushall } + Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping) end after do - Gitlab::RedisCaching.with { |conn| conn.flushall } - Gitlab::RedisQueues.with { |conn| conn.flushall } - Gitlab::RedisSharedState.with { |conn| conn.flushall } + Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } end it 'does not perform housekeeping when not needed' do diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 0076cc74f129..38ab3d7d70a1 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::RedisSharedState::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::Redis::SharedState::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Gitlab::RedisSharedState::CannotConnectError + rescue Gitlab::Redis::SharedState::CannotConnectError end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8b56d667fce7..02d5e7a32520 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,7 @@ require './spec/simplecov_env' -require './lib/gitlab/redis_caching' -require './lib/gitlab/redis_queues' -require './lib/gitlab/redis_shared_state' +require './lib/gitlab/redis/caching' +require './lib/gitlab/redis/queues' +require './lib/gitlab/redis/shared_state' SimpleCovEnv.start! ENV["RAILS_ENV"] ||= 'test' @@ -89,20 +89,20 @@ end config.around(:each, :clean_gitlab_redis_caching) do |example| - Gitlab::RedisCaching.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) example.run - Gitlab::RedisCaching.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) end config.around(:each, :clean_gitlab_redis_shared_state) do |example| - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) example.run - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) end end diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 3a72b2c41a71..6e25c5361916 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,9 +1,9 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::RedisCaching.with(&:flushall) - Gitlab::RedisQueues.with(&:flushall) - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Queues.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) end before do -- GitLab From 09fd1bc29e8ff88b4dba6e5266e509f130c2bf62 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 08:36:10 +0800 Subject: [PATCH 08/62] create Redis module for health checks --- .../health_checks/redis/caching_check.rb | 27 +++++++++++++++++++ .../health_checks/redis/queues_check.rb | 27 +++++++++++++++++++ .../health_checks/redis/shared_state_check.rb | 27 +++++++++++++++++++ .../health_checks/redis_caching_check.rb | 25 ----------------- .../health_checks/redis_queues_check.rb | 25 ----------------- .../health_checks/redis_shared_state_check.rb | 25 ----------------- .../caching_check_spec.rb} | 4 +-- .../queues_check_spec.rb} | 4 +-- .../shared_state_check_spec.rb} | 4 +-- spec/spec_helper.rb | 3 --- 10 files changed, 87 insertions(+), 84 deletions(-) create mode 100644 lib/gitlab/health_checks/redis/caching_check.rb create mode 100644 lib/gitlab/health_checks/redis/queues_check.rb create mode 100644 lib/gitlab/health_checks/redis/shared_state_check.rb delete mode 100644 lib/gitlab/health_checks/redis_caching_check.rb delete mode 100644 lib/gitlab/health_checks/redis_queues_check.rb delete mode 100644 lib/gitlab/health_checks/redis_shared_state_check.rb rename spec/lib/gitlab/health_checks/{redis_caching_check_spec.rb => redis/caching_check_spec.rb} (52%) rename spec/lib/gitlab/health_checks/{redis_queues_check_spec.rb => redis/queues_check_spec.rb} (52%) rename spec/lib/gitlab/health_checks/{redis_shared_state_check_spec.rb => redis/shared_state_check_spec.rb} (53%) diff --git a/lib/gitlab/health_checks/redis/caching_check.rb b/lib/gitlab/health_checks/redis/caching_check.rb new file mode 100644 index 000000000000..a3c1780e64bc --- /dev/null +++ b/lib/gitlab/health_checks/redis/caching_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class CachingCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_caching_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Caching.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 000000000000..c61033ae5721 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 000000000000..edd55400c183 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_caching_check.rb b/lib/gitlab/health_checks/redis_caching_check.rb deleted file mode 100644 index 767c5adddf1c..000000000000 --- a/lib/gitlab/health_checks/redis_caching_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCachingCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_caching_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis::Caching.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis_queues_check.rb b/lib/gitlab/health_checks/redis_queues_check.rb deleted file mode 100644 index b6ce6fe82e81..000000000000 --- a/lib/gitlab/health_checks/redis_queues_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisQueuesCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_queues_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis::Queues.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis_shared_state_check.rb b/lib/gitlab/health_checks/redis_shared_state_check.rb deleted file mode 100644 index a478576ba8d4..000000000000 --- a/lib/gitlab/health_checks/redis_shared_state_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisSharedStateCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_shared_state_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis::SharedState.with(&:ping) - end - end - end - end - end -end diff --git a/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb similarity index 52% rename from spec/lib/gitlab/health_checks/redis_caching_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/caching_check_spec.rb index 710028a4ec8d..0e66bd04303f 100644 --- a/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisCachingCheck do +describe Gitlab::HealthChecks::Redis::CachingCheck do include_examples 'simple_check', 'redis_caching_ping', 'RedisCaching', 'PONG' end diff --git a/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb similarity index 52% rename from spec/lib/gitlab/health_checks/redis_queues_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/queues_check_spec.rb index 52727f5a8b34..c69443d205d0 100644 --- a/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisQueuesCheck do +describe Gitlab::HealthChecks::Redis::QueuesCheck do include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG' end diff --git a/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb similarity index 53% rename from spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb index 8b34a9e07083..b72e152bbe2f 100644 --- a/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisSharedStateCheck do +describe Gitlab::HealthChecks::Redis::SharedStateCheck do include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG' end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 02d5e7a32520..b90c94f7ae87 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,4 @@ require './spec/simplecov_env' -require './lib/gitlab/redis/caching' -require './lib/gitlab/redis/queues' -require './lib/gitlab/redis/shared_state' SimpleCovEnv.start! ENV["RAILS_ENV"] ||= 'test' -- GitLab From ccebd3e90aa7295f6fcdc68859bbb3198721bd7b Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 08:38:47 +0800 Subject: [PATCH 09/62] fix for static analysis - require newline at end of file --- lib/gitlab/redis/queues.rb | 2 +- lib/gitlab/redis/wrapper.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index d7e8d9378f97..ff58556874df 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -22,4 +22,4 @@ def default_url end end end -end \ No newline at end of file +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c069ac7da4a8..401559aa3873 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -96,4 +96,4 @@ def fetch_config end end end -end \ No newline at end of file +end -- GitLab From 80a4d17899a04902d88e2c2ada38131f294e66a4 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 08:57:19 +0800 Subject: [PATCH 10/62] Health Checks to use Redis module --- app/controllers/health_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 4c71f2d70dc4..c1421de14fda 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,9 +4,9 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCachingCheck, - Gitlab::HealthChecks::RedisQueuesCheck, - Gitlab::HealthChecks::RedisSharedStateCheck, + Gitlab::HealthChecks::Redis::CachingCheck, + Gitlab::HealthChecks::Redis::QueuesCheck, + Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze -- GitLab From 7715bb9aeeb5122214d694e935086b7f51400b01 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 09:46:08 +0800 Subject: [PATCH 11/62] add redis module for rspec health checks --- spec/controllers/health_controller_spec.rb | 12 ++++++------ .../{redis_caching_spec.rb => redis/caching_spec.rb} | 0 .../{redis_queues_spec.rb => redis/queues_spec.rb} | 0 .../shared_state_spec.rb} | 0 4 files changed, 6 insertions(+), 6 deletions(-) rename spec/lib/gitlab/{redis_caching_spec.rb => redis/caching_spec.rb} (100%) rename spec/lib/gitlab/{redis_queues_spec.rb => redis/queues_spec.rb} (100%) rename spec/lib/gitlab/{redis_shared_state_spec.rb => redis/shared_state_spec.rb} (100%) diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index 24d39ee64685..13cb333be2c5 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,9 +19,9 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_caching_check']['status']).to eq('ok') - expect(json_response['redis_queues_check']['status']).to eq('ok') - expect(json_response['redis_shared_state_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') end @@ -44,9 +44,9 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_caching_check']['status']).to eq('ok') - expect(json_response['redis_queues_check']['status']).to eq('ok') - expect(json_response['redis_shared_state_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') end end diff --git a/spec/lib/gitlab/redis_caching_spec.rb b/spec/lib/gitlab/redis/caching_spec.rb similarity index 100% rename from spec/lib/gitlab/redis_caching_spec.rb rename to spec/lib/gitlab/redis/caching_spec.rb diff --git a/spec/lib/gitlab/redis_queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb similarity index 100% rename from spec/lib/gitlab/redis_queues_spec.rb rename to spec/lib/gitlab/redis/queues_spec.rb diff --git a/spec/lib/gitlab/redis_shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb similarity index 100% rename from spec/lib/gitlab/redis_shared_state_spec.rb rename to spec/lib/gitlab/redis/shared_state_spec.rb -- GitLab From 4661e3da4c695dcc560742c5339594cd6542b312 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 11:29:05 +0800 Subject: [PATCH 12/62] make resque.yml the default config file for new redis classes; add new test cases for that (more tests needed) --- config/resque.yml.example | 40 ++++ lib/gitlab/redis/caching.rb | 4 +- lib/gitlab/redis/queues.rb | 4 +- lib/gitlab/redis/shared_state.rb | 4 +- lib/gitlab/redis/wrapper.rb | 12 +- .../fixtures/config/redis_config_with_env.yml | 2 + .../fixtures/config/redis_new_format_host.yml | 29 +++ .../config/redis_new_format_socket.yml | 6 + .../fixtures/config/redis_old_format_host.yml | 5 + .../config/redis_old_format_socket.yml | 3 + spec/lib/gitlab/redis/wrapper_spec.rb | 211 ++++++++++++++++++ 11 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 config/resque.yml.example create mode 100644 spec/fixtures/config/redis_config_with_env.yml create mode 100644 spec/fixtures/config/redis_new_format_host.yml create mode 100644 spec/fixtures/config/redis_new_format_socket.yml create mode 100644 spec/fixtures/config/redis_old_format_host.yml create mode 100644 spec/fixtures/config/redis_old_format_socket.yml create mode 100644 spec/lib/gitlab/redis/wrapper_spec.rb diff --git a/config/resque.yml.example b/config/resque.yml.example new file mode 100644 index 000000000000..e983be5b32e4 --- /dev/null +++ b/config/resque.yml.example @@ -0,0 +1,40 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6379 + # sentinels: + # - + # host: localhost + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port +test: + url: redis://localhost:6379 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6379 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.caching.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6379 + # sentinels: + # - + # host: slave1 + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb index e4214fe761d7..04b361ec1671 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/caching.rb @@ -13,7 +13,9 @@ class << self end def config_file - ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || File.expand_path('../../../config/redis.caching.yml', __dir__) + ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || + File.expand_path('../../../config/redis.caching.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index ff58556874df..32676f2172d9 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -14,7 +14,9 @@ class << self end def config_file - ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || File.expand_path('../../../config/redis.queues.yml', __dir__) + ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || + File.expand_path('../../../config/redis.queues.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 23e1564fb65f..aedc51af838a 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -13,7 +13,9 @@ class << self end def config_file - ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../../config/redis.shared_state.yml', __dir__) + ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || + File.expand_path('../../../config/redis.shared_state.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 401559aa3873..31e8dd03d31f 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -10,9 +10,10 @@ module Redis class Wrapper BaseError = ::Redis::BaseError.freeze CannotConnectError = ::Redis::CannotConnectError.freeze + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze class << self - delegate :params, :url, to: :new + delegate :_raw_config, :fetch_configm, :params, :url, to: :new def with @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @@ -94,6 +95,15 @@ def raw_config_hash def fetch_config self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end + + def config_file + ENV['GITLAB_REDIS_CONFIG_FILE'] || + File.expand_path('../../../resque.yml', __dir__) + end + + def default_url + DEFAULT_REDIS_URL + end end end end diff --git a/spec/fixtures/config/redis_config_with_env.yml b/spec/fixtures/config/redis_config_with_env.yml new file mode 100644 index 000000000000..f5860f37e475 --- /dev/null +++ b/spec/fixtures/config/redis_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_URL'] %> diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/redis_new_format_host.yml new file mode 100644 index 000000000000..8d134d467e9b --- /dev/null +++ b/spec/fixtures/config/redis_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: slave1 + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_new_format_socket.yml b/spec/fixtures/config/redis_new_format_socket.yml new file mode 100644 index 000000000000..4e76830c281d --- /dev/null +++ b/spec/fixtures/config/redis_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.sock +test: + url: unix:/path/to/redis.sock +production: + url: unix:/path/to/redis.sock diff --git a/spec/fixtures/config/redis_old_format_host.yml b/spec/fixtures/config/redis_old_format_host.yml new file mode 100644 index 000000000000..253d0a994f5e --- /dev/null +++ b/spec/fixtures/config/redis_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6379/99 +test: redis://:mypassword@localhost:6379/99 +production: redis://:mypassword@localhost:6379/99 diff --git a/spec/fixtures/config/redis_old_format_socket.yml b/spec/fixtures/config/redis_old_format_socket.yml new file mode 100644 index 000000000000..fd31ce8ea3d7 --- /dev/null +++ b/spec/fixtures/config/redis_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.sock +test: unix:/path/to/old/redis.sock +production: unix:/path/to/old/redis.sock diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb new file mode 100644 index 000000000000..214361c6ece9 --- /dev/null +++ b/spec/lib/gitlab/redis/wrapper_spec.rb @@ -0,0 +1,211 @@ +require 'spec_helper' + +describe Gitlab::Redis::Wrapper do + include StubENV + + let(:config) { 'config/resque.yml' } + + before(:each) do + stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s) + clear_raw_config + end + + after(:each) do + clear_raw_config + end + + describe '.params' do + subject { described_class.params } + + it 'withstands mutation' do + params1 = described_class.params + params2 = described_class.params + params1[:foo] = :bar + + expect(params2).not_to have_key(:foo) + end + + context 'when url contains unix socket reference' do + let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/old/redis.sock') + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/redis.sock') + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' + + expect(url2).not_to end_with('foobar') + end + + context 'when yml file with env variable' do + let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' } + + before do + stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379') + end + + it 'reads redis url from env variable' do + expect(described_class.url).to eq 'redis://redishost:6379' + end + end + end + + describe '._raw_config' do + subject { described_class._raw_config } + let(:config) { '/var/empty/doesnotexist' } + + it 'should be frozen' do + expect(subject).to be_frozen + end + + it 'returns false when the file does not exist' do + expect(subject).to eq(false) + end + end + + describe '.with' do + before { clear_pool } + after { clear_pool } + + context 'when running not on sidekiq workers' do + before { allow(Sidekiq).to receive(:server?).and_return(false) } + + it 'instantiates a connection pool with size 5' do + expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original + + described_class.with { |_wrapper| true } + end + end + + context 'when running on sidekiq workers' do + before do + allow(Sidekiq).to receive(:server?).and_return(true) + allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) + end + + it 'instantiates a connection pool with a size based on the concurrency of the worker' do + expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original + + described_class.with { |_wrapper| true } + end + end + end + + describe '#sentinels' do + subject { described_class.new(Rails.env).sentinels } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + + it 'returns an array of hashes with host and port keys' do + is_expected.to include(host: 'localhost', port: 26379) + is_expected.to include(host: 'slave2', port: 26379) + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#sentinels?' do + subject { described_class.new(Rails.env).sentinels? } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + + it 'returns true' do + is_expected.to be_truthy + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + + it 'returns false' do + is_expected.to be_falsey + end + end + end + + describe '#raw_config_hash' do + it 'returns default redis url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') + end + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(described_class).to receive(:_raw_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end + + def clear_raw_config + described_class.remove_instance_variable(:@_raw_config) + rescue NameError + # raised if @_raw_config was not set; ignore + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end +end -- GitLab From ffe3e5edb312341fef68d2117850ee27d1abf67e Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 12:19:15 +0800 Subject: [PATCH 13/62] static analysis - code style formatting --- lib/gitlab/redis/caching.rb | 4 ++-- lib/gitlab/redis/queues.rb | 4 ++-- lib/gitlab/redis/shared_state.rb | 4 ++-- lib/gitlab/redis/wrapper.rb | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb index 04b361ec1671..2523e5acd9cc 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/caching.rb @@ -14,8 +14,8 @@ class << self def config_file ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || - File.expand_path('../../../config/redis.caching.yml', __dir__) || - superclass.config_file + File.expand_path('../../../config/redis.caching.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 32676f2172d9..7ae3a4810262 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -15,8 +15,8 @@ class << self def config_file ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || - File.expand_path('../../../config/redis.queues.yml', __dir__) || - superclass.config_file + File.expand_path('../../../config/redis.queues.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index aedc51af838a..0f101e852ddf 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -14,8 +14,8 @@ class << self def config_file ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || - File.expand_path('../../../config/redis.shared_state.yml', __dir__) || - superclass.config_file + File.expand_path('../../../config/redis.shared_state.yml', __dir__) || + superclass.config_file end def default_url diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 31e8dd03d31f..787c08d3e4ca 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -98,7 +98,7 @@ def fetch_config def config_file ENV['GITLAB_REDIS_CONFIG_FILE'] || - File.expand_path('../../../resque.yml', __dir__) + File.expand_path('../../../resque.yml', __dir__) end def default_url -- GitLab From 446833e6e7a69e1d2b9fc3a20b0d18f351f4dbe7 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 15:50:48 +0800 Subject: [PATCH 14/62] make `resque.yml` the default if no other config file exists --- lib/gitlab/redis/caching.rb | 12 ++++++++---- lib/gitlab/redis/queues.rb | 12 ++++++++---- lib/gitlab/redis/shared_state.rb | 12 ++++++++---- lib/gitlab/redis/wrapper.rb | 33 +++++++++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb index 2523e5acd9cc..72eb5f633688 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/caching.rb @@ -7,15 +7,19 @@ module Redis class Caching < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze + ENV_VAR_NAME = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze + CONFIG_FILE_NAME = 'redis.caching.yml'.freeze class << self delegate :_raw_config, :fetch_config, to: :new end - def config_file - ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || - File.expand_path('../../../config/redis.caching.yml', __dir__) || - superclass.config_file + def env_var_for_config_file_name + ENV_VAR_NAME + end + + def config_file_name + CONFIG_FILE_NAME end def default_url diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 7ae3a4810262..a4f752f5bd6b 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,15 +8,19 @@ class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze + ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + CONFIG_FILE_NAME = 'redis.queues.yml'.freeze class << self delegate :_raw_config, :fetch_config, to: :new end - def config_file - ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || - File.expand_path('../../../config/redis.queues.yml', __dir__) || - superclass.config_file + def env_var_for_config_file_name + ENV_VAR_NAME + end + + def config_file_name + CONFIG_FILE_NAME end def default_url diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 0f101e852ddf..eb5df79a316f 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,15 +7,19 @@ module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + CONFIG_FILE_NAME = 'redis.shared_state.yml'.freeze class << self delegate :_raw_config, :fetch_config, to: :new end - def config_file - ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || - File.expand_path('../../../config/redis.shared_state.yml', __dir__) || - superclass.config_file + def env_var_for_config_file_name + ENV_VAR_NAME + end + + def config_file_name + CONFIG_FILE_NAME end def default_url diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 787c08d3e4ca..33d805d0451e 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -11,6 +11,8 @@ class Wrapper BaseError = ::Redis::BaseError.freeze CannotConnectError = ::Redis::CannotConnectError.freeze DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + CONFIG_FILE_NAME = 'resque.yml'.freeze class << self delegate :_raw_config, :fetch_configm, :params, :url, to: :new @@ -88,7 +90,7 @@ def raw_config_hash if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: default_url } + { url: "#{default_url}" } end end @@ -96,9 +98,34 @@ def fetch_config self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end + def env_var_for_config_file_name + ENV_VAR_NAME + end + + def config_file_name + CONFIG_FILE_NAME + end + def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || - File.expand_path('../../../resque.yml', __dir__) + # if ENV set for sub-class, use it even if it points to a file does not exist + proposed_file_name = ENV[env_var_for_config_file_name] + file_name = proposed_file_name if proposed_file_name + # otherwise, if config files exists for sub-class, use it + if !file_name + proposed_file_name = File.expand_path("../../../config/#{config_file_name}", __dir__) + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + if !file_name + proposed_file_name = ENV[ENV_VAR_NAME] + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + # otherwise, if config files exists for wrapper class, use it + if !file_name + proposed_file_name = File.expand_path("../../../config/#{CONFIG_FILE_NAME}", __dir__) + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + file_name end def default_url -- GitLab From 3781b9bdc28598cb513722dd9abc115daad0c9c9 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 17:14:56 +0800 Subject: [PATCH 15/62] make `resque.yml` the default if no other config file exists --- .flayignore | 1 + lib/gitlab/redis/caching.rb | 21 +++++++------------- lib/gitlab/redis/queues.rb | 21 +++++++------------- lib/gitlab/redis/shared_state.rb | 21 +++++++------------- lib/gitlab/redis/wrapper.rb | 23 +++++++--------------- spec/lib/gitlab/redis/caching_spec.rb | 2 +- spec/lib/gitlab/redis/queues_spec.rb | 2 +- spec/lib/gitlab/redis/shared_state_spec.rb | 2 +- 8 files changed, 32 insertions(+), 61 deletions(-) diff --git a/.flayignore b/.flayignore index 47597025115c..e2d0a2e50c55 100644 --- a/.flayignore +++ b/.flayignore @@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb lib/gitlab/diff/position_tracer.rb app/policies/project_policy.rb app/models/concerns/relative_positioning.rb +lib/gitlab/redis/*.rb diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb index 72eb5f633688..b9a0149cf880 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/caching.rb @@ -6,24 +6,17 @@ module Gitlab module Redis class Caching < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze - ENV_VAR_NAME = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze - CONFIG_FILE_NAME = 'redis.caching.yml'.freeze + DEFAULT_REDIS_CACHING_URL = 'redis://localhost:6380'.freeze class << self - delegate :_raw_config, :fetch_config, to: :new + delegate :_raw_config, :fetch_config, :config_file, to: :new end - def env_var_for_config_file_name - ENV_VAR_NAME - end - - def config_file_name - CONFIG_FILE_NAME - end - - def default_url - DEFAULT_CACHING_URL + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_CACHING_URL + @environment_var_name = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze + @config_file_name = 'redis.caching.yml'.freeze end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index a4f752f5bd6b..b45a6a757819 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -7,24 +7,17 @@ module Redis class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze - ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - CONFIG_FILE_NAME = 'redis.queues.yml'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze class << self - delegate :_raw_config, :fetch_config, to: :new + delegate :_raw_config, :fetch_config, :config_file, to: :new end - def env_var_for_config_file_name - ENV_VAR_NAME - end - - def config_file_name - CONFIG_FILE_NAME - end - - def default_url - DEFAULT_QUEUES_URL + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_QUEUES_URL + @environment_var_name = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + @config_file_name = 'redis.queues.yml'.freeze end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index eb5df79a316f..89fae79e248a 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -6,24 +6,17 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze - DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - CONFIG_FILE_NAME = 'redis.shared_state.yml'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze class << self - delegate :_raw_config, :fetch_config, to: :new + delegate :_raw_config, :fetch_config, :config_file, to: :new end - def env_var_for_config_file_name - ENV_VAR_NAME - end - - def config_file_name - CONFIG_FILE_NAME - end - - def default_url - DEFAULT_SHARED_STATE_URL + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_SHARED_STATE_URL + @environment_var_name = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + @config_file_name = 'redis.shared_state.yml'.freeze end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 33d805d0451e..78f00d26585d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -15,7 +15,7 @@ class Wrapper CONFIG_FILE_NAME = 'resque.yml'.freeze class << self - delegate :_raw_config, :fetch_configm, :params, :url, to: :new + delegate :_raw_config, :fetch_config, :config_file, :params, :url, to: :new def with @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @@ -47,6 +47,9 @@ def _raw_config def initialize(rails_env = nil) @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_URL + @environment_var_name = ENV_VAR_NAME + @config_file_name = CONFIG_FILE_NAME end def params @@ -90,7 +93,7 @@ def raw_config_hash if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: "#{default_url}" } + { url: "#{@default_url}" } end end @@ -98,21 +101,13 @@ def fetch_config self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end - def env_var_for_config_file_name - ENV_VAR_NAME - end - - def config_file_name - CONFIG_FILE_NAME - end - def config_file # if ENV set for sub-class, use it even if it points to a file does not exist - proposed_file_name = ENV[env_var_for_config_file_name] + proposed_file_name = ENV[@environment_var_name] file_name = proposed_file_name if proposed_file_name # otherwise, if config files exists for sub-class, use it if !file_name - proposed_file_name = File.expand_path("../../../config/#{config_file_name}", __dir__) + proposed_file_name = File.expand_path("../../../config/#{@config_file_name}", __dir__) file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) end # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist @@ -127,10 +122,6 @@ def config_file end file_name end - - def default_url - DEFAULT_REDIS_URL - end end end end diff --git a/spec/lib/gitlab/redis/caching_spec.rb b/spec/lib/gitlab/redis/caching_spec.rb index a62e2c80fff6..1ff8dc1e78a8 100644 --- a/spec/lib/gitlab/redis/caching_spec.rb +++ b/spec/lib/gitlab/redis/caching_spec.rb @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_REDIS_CACHING_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index b38157b0d31a..d4878349edd1 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -180,7 +180,7 @@ it 'returns default queues url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Queues::DEFAULT_QUEUES_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb index 67ec76ad7052..978b6c6096ef 100644 --- a/spec/lib/gitlab/redis/shared_state_spec.rb +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -180,7 +180,7 @@ it 'returns default redis_shared_state url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::SharedState::DEFAULT_SHARED_STATE_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL) end it 'returns old-style single url config in a hash' do -- GitLab From cfd4754184cbd0a8e5e1477be1787c2fe112604b Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 17:52:06 +0800 Subject: [PATCH 16/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- .flayignore | 1 + app/controllers/health_controller.rb | 6 +- app/models/ci/runner.rb | 2 +- app/models/project.rb | 6 +- config/application.rb | 10 +- config/initializers/7_redis.rb | 6 +- config/initializers/session_store.rb | 4 +- config/initializers/sidekiq.rb | 6 +- config/mail_room.yml | 4 +- config/resque.yml.example | 40 ++++ ...er_activities_to_users_last_activity_on.rb | 4 +- lib/gitlab/auth/unique_ips_limiter.rb | 2 +- .../cache/ci/project_pipeline_status.rb | 10 +- lib/gitlab/chat_name_token.rb | 6 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/etag_caching/store.rb | 4 +- lib/gitlab/exclusive_lease.rb | 6 +- .../health_checks/redis/caching_check.rb | 27 +++ .../health_checks/redis/queues_check.rb | 27 +++ .../health_checks/redis/shared_state_check.rb | 27 +++ .../health_checks/redis_caching_check.rb | 25 --- .../health_checks/redis_queues_check.rb | 25 --- .../health_checks/redis_shared_state_check.rb | 25 --- lib/gitlab/lfs_token.rb | 2 +- lib/gitlab/mail_room.rb | 4 +- lib/gitlab/redis/caching.rb | 23 ++ lib/gitlab/redis/queues.rb | 24 ++ lib/gitlab/redis/shared_state.rb | 23 ++ lib/gitlab/redis/wrapper.rb | 127 +++++++++++ lib/gitlab/redis_caching.rb | 22 -- lib/gitlab/redis_queues.rb | 23 -- lib/gitlab/redis_shared_state.rb | 22 -- lib/gitlab/redis_wrapper.rb | 97 -------- lib/gitlab/user_activities.rb | 6 +- lib/gitlab/workhorse.rb | 2 +- lib/tasks/cache.rake | 6 +- spec/config/mail_room_spec.rb | 4 +- spec/controllers/health_controller_spec.rb | 12 +- .../fixtures/config/redis_config_with_env.yml | 2 + .../fixtures/config/redis_new_format_host.yml | 29 +++ .../config/redis_new_format_socket.yml | 6 + .../fixtures/config/redis_old_format_host.yml | 5 + .../config/redis_old_format_socket.yml | 3 + .../cache/ci/project_pipeline_status_spec.rb | 22 +- spec/lib/gitlab/current_settings_spec.rb | 2 +- .../caching_check_spec.rb} | 4 +- .../queues_check_spec.rb} | 4 +- .../shared_state_check_spec.rb} | 4 +- .../caching_spec.rb} | 4 +- .../queues_spec.rb} | 4 +- .../shared_state_spec.rb} | 4 +- spec/lib/gitlab/redis/wrapper_spec.rb | 211 ++++++++++++++++++ spec/lib/gitlab/user_activities_spec.rb | 20 +- ...tivities_to_users_last_activity_on_spec.rb | 2 +- spec/models/ci/runner_spec.rb | 4 +- spec/services/git_push_service_spec.rb | 12 +- .../services/projects/destroy_service_spec.rb | 4 +- spec/spec_helper.rb | 11 +- .../unique_ip_check_shared_examples.rb | 6 +- 59 files changed, 684 insertions(+), 351 deletions(-) create mode 100644 config/resque.yml.example create mode 100644 lib/gitlab/health_checks/redis/caching_check.rb create mode 100644 lib/gitlab/health_checks/redis/queues_check.rb create mode 100644 lib/gitlab/health_checks/redis/shared_state_check.rb delete mode 100644 lib/gitlab/health_checks/redis_caching_check.rb delete mode 100644 lib/gitlab/health_checks/redis_queues_check.rb delete mode 100644 lib/gitlab/health_checks/redis_shared_state_check.rb create mode 100644 lib/gitlab/redis/caching.rb create mode 100644 lib/gitlab/redis/queues.rb create mode 100644 lib/gitlab/redis/shared_state.rb create mode 100644 lib/gitlab/redis/wrapper.rb delete mode 100644 lib/gitlab/redis_caching.rb delete mode 100644 lib/gitlab/redis_queues.rb delete mode 100644 lib/gitlab/redis_shared_state.rb delete mode 100644 lib/gitlab/redis_wrapper.rb create mode 100644 spec/fixtures/config/redis_config_with_env.yml create mode 100644 spec/fixtures/config/redis_new_format_host.yml create mode 100644 spec/fixtures/config/redis_new_format_socket.yml create mode 100644 spec/fixtures/config/redis_old_format_host.yml create mode 100644 spec/fixtures/config/redis_old_format_socket.yml rename spec/lib/gitlab/health_checks/{redis_caching_check_spec.rb => redis/caching_check_spec.rb} (52%) rename spec/lib/gitlab/health_checks/{redis_queues_check_spec.rb => redis/queues_check_spec.rb} (52%) rename spec/lib/gitlab/health_checks/{redis_shared_state_check_spec.rb => redis/shared_state_check_spec.rb} (53%) rename spec/lib/gitlab/{redis_caching_spec.rb => redis/caching_spec.rb} (98%) rename spec/lib/gitlab/{redis_queues_spec.rb => redis/queues_spec.rb} (98%) rename spec/lib/gitlab/{redis_shared_state_spec.rb => redis/shared_state_spec.rb} (98%) create mode 100644 spec/lib/gitlab/redis/wrapper_spec.rb diff --git a/.flayignore b/.flayignore index 47597025115c..e2d0a2e50c55 100644 --- a/.flayignore +++ b/.flayignore @@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb lib/gitlab/diff/position_tracer.rb app/policies/project_policy.rb app/models/concerns/relative_positioning.rb +lib/gitlab/redis/*.rb diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 4c71f2d70dc4..c1421de14fda 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,9 +4,9 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCachingCheck, - Gitlab::HealthChecks::RedisQueuesCheck, - Gitlab::HealthChecks::RedisSharedStateCheck, + Gitlab::HealthChecks::Redis::CachingCheck, + Gitlab::HealthChecks::Redis::QueuesCheck, + Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 4907f65e2d5a..d2be57e36ff0 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,7 +145,7 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| redis_queues.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index eb16af80fbf7..16bdde93c096 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1280,15 +1280,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) diff --git a/config/application.rb b/config/application.rb index de565d2edacd..af78c06e3219 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,9 +6,9 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis_caching') - require_dependency Rails.root.join('lib/gitlab/redis_queues') - require_dependency Rails.root.join('lib/gitlab/redis_shared_state') + require_dependency Rails.root.join('lib/gitlab/redis/caching') + require_dependency Rails.root.join('lib/gitlab/redis/queues') + require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -141,8 +141,8 @@ class Application < Rails::Application end # Use caching across all environments - caching_config_hash = Gitlab::RedisCaching.params - caching_config_hash[:namespace] = Gitlab::RedisCaching::CACHE_NAMESPACE + caching_config_hash = Gitlab::Redis::Caching.params + caching_config_hash[:namespace] = Gitlab::Redis::Caching::CACHE_NAMESPACE caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 797361b5968f..7ad600aa26ae 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::RedisCaching.with { nil } -Gitlab::RedisQueues.with { nil } -Gitlab::RedisSharedState.with { nil } +Gitlab::Redis::Caching.with { nil } +Gitlab::Redis::Queues.with { nil } +Gitlab::Redis::SharedState.with { nil } diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 22256d0ef13b..87768095859b 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,8 +13,8 @@ if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - sessions_config = Gitlab::RedisSharedState.params - sessions_config[:namespace] = Gitlab::RedisSharedState::SESSION_NAMESPACE + sessions_config = Gitlab::Redis::SharedState.params + sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 27e18fa960f7..bf99b248bbcb 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,6 +1,6 @@ # Custom Queues configuration -queues_config_hash = Gitlab::RedisQueues.params -queues_config_hash[:namespace] = Gitlab::RedisQueues::SIDEKIQ_NAMESPACE +queues_config_hash = Gitlab::Redis::Queues.params +queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } @@ -74,5 +74,5 @@ end end end -rescue Gitlab::RedisQueues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue Gitlab::Redis::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/config/mail_room.yml b/config/mail_room.yml index d191400e8105..c3a5be8d38c5 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::RedisQueues::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::RedisQueues::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/config/resque.yml.example b/config/resque.yml.example new file mode 100644 index 000000000000..e983be5b32e4 --- /dev/null +++ b/config/resque.yml.example @@ -0,0 +1,40 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6379 + # sentinels: + # - + # host: localhost + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port +test: + url: redis://localhost:6379 + # + # notes for when CI supports service configuration on shared image + # url: redis://localhost:6379 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.caching.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6379 + # sentinels: + # - + # host: slave1 + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb index f8e0895a53f8..e30b8788234c 100644 --- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -56,7 +56,7 @@ def users_table end def activities(from, to, page: 1) - Gitlab::RedisSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, with_scores: true, limit: limit(page)) @@ -64,7 +64,7 @@ def activities(from, to, page: 1) end def activities_count(from, to) - Gitlab::RedisSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) end end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 7b82521cdd4e..6fff4309e893 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| unique_ips_count = nil redis_shared_state.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 414e994c8f24..492e1254f164 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::RedisCaching.with do |redis_caching| + result = Gitlab::Redis::Caching.with do |redis_caching| redis_caching.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| self.sha, self.status, self.ref = redis_caching.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.del(cache_key) end end @@ -120,7 +120,7 @@ def delete_from_cache def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index d4a858dbdd4c..bf8d442140fc 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,14 +12,14 @@ def initialize(token = new_token) end def get - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| data = redis_shared_state.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| params = params.to_json redis_shared_state.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token @@ -27,7 +27,7 @@ def store!(params) end def delete - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.del(redis_shared_state_key) end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9d07afc29a19..88fd408ef475 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::RedisCaching::BaseError, ::Errno::ENOENT + rescue ::Gitlab::Redis::Caching::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 587580102ed9..7096be42beab 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -5,13 +5,13 @@ class Store SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::RedisSharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } + Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a2c6a2933cbd..19f3b7b50b74 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,7 +18,7 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end @@ -37,14 +37,14 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.exists(@redis_shared_state_key) end end diff --git a/lib/gitlab/health_checks/redis/caching_check.rb b/lib/gitlab/health_checks/redis/caching_check.rb new file mode 100644 index 000000000000..a3c1780e64bc --- /dev/null +++ b/lib/gitlab/health_checks/redis/caching_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class CachingCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_caching_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Caching.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 000000000000..c61033ae5721 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 000000000000..edd55400c183 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_caching_check.rb b/lib/gitlab/health_checks/redis_caching_check.rb deleted file mode 100644 index 5a56cc37246f..000000000000 --- a/lib/gitlab/health_checks/redis_caching_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCachingCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_caching_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::RedisCaching.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis_queues_check.rb b/lib/gitlab/health_checks/redis_queues_check.rb deleted file mode 100644 index 2b0d270fb587..000000000000 --- a/lib/gitlab/health_checks/redis_queues_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisQueuesCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_queues_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::RedisQueues.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis_shared_state_check.rb b/lib/gitlab/health_checks/redis_shared_state_check.rb deleted file mode 100644 index 68a5c4fce593..000000000000 --- a/lib/gitlab/health_checks/redis_shared_state_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisSharedStateCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_shared_state_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::RedisSharedState.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 4a837401592c..043bfe18545b 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,7 +18,7 @@ def initialize(actor) end def token - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| token = redis_shared_state.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) redis_shared_state.set(redis_shared_state_key, token, ex: EXPIRY_TIME) diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 88eb0afa91ab..a2b363e54085 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis_queues' unless defined?(Gitlab::RedisQueues) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,7 +34,7 @@ def fetch_config config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis_queues = Gitlab::RedisQueues.new(rails_env) + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) config[:redis_url] = gitlab_redis_queues.url # @@ TODO this is too-much-information on topology in the client of an object diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/caching.rb new file mode 100644 index 000000000000..b9a0149cf880 --- /dev/null +++ b/lib/gitlab/redis/caching.rb @@ -0,0 +1,23 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class Caching < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHING_URL = 'redis://localhost:6380'.freeze + + class << self + delegate :_raw_config, :fetch_config, :config_file, to: :new + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_CACHING_URL + @environment_var_name = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze + @config_file_name = 'redis.caching.yml'.freeze + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 000000000000..b45a6a757819 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,24 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + + class << self + delegate :_raw_config, :fetch_config, :config_file, to: :new + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_QUEUES_URL + @environment_var_name = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + @config_file_name = 'redis.queues.yml'.freeze + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 000000000000..89fae79e248a --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,23 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require_relative 'wrapper' + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + + class << self + delegate :_raw_config, :fetch_config, :config_file, to: :new + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_SHARED_STATE_URL + @environment_var_name = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + @config_file_name = 'redis.shared_state.yml'.freeze + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 000000000000..78f00d26585d --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,127 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'rails' +require 'redis' +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + BaseError = ::Redis::BaseError.freeze + CannotConnectError = ::Redis::CannotConnectError.freeze + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + CONFIG_FILE_NAME = 'resque.yml'.freeze + + class << self + delegate :_raw_config, :fetch_config, :config_file, :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(config_file)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + @default_url = DEFAULT_REDIS_URL + @environment_var_name = ENV_VAR_NAME + @config_file_name = CONFIG_FILE_NAME + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: "#{@default_url}" } + end + end + + def fetch_config + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false + end + + def config_file + # if ENV set for sub-class, use it even if it points to a file does not exist + proposed_file_name = ENV[@environment_var_name] + file_name = proposed_file_name if proposed_file_name + # otherwise, if config files exists for sub-class, use it + if !file_name + proposed_file_name = File.expand_path("../../../config/#{@config_file_name}", __dir__) + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + if !file_name + proposed_file_name = ENV[ENV_VAR_NAME] + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + # otherwise, if config files exists for wrapper class, use it + if !file_name + proposed_file_name = File.expand_path("../../../config/#{CONFIG_FILE_NAME}", __dir__) + file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) + end + file_name + end + end + end +end diff --git a/lib/gitlab/redis_caching.rb b/lib/gitlab/redis_caching.rb deleted file mode 100644 index 9636c1219948..000000000000 --- a/lib/gitlab/redis_caching.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisCaching < ::Gitlab::RedisWrapper - CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_CACHING_URL = 'redis://localhost:6380'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_CACHING_CONFIG_FILE'] || File.expand_path('../../config/redis.caching.yml', __dir__) - end - - def default_url - DEFAULT_CACHING_URL - end - end -end diff --git a/lib/gitlab/redis_queues.rb b/lib/gitlab/redis_queues.rb deleted file mode 100644 index 3cdd9cde17cf..000000000000 --- a/lib/gitlab/redis_queues.rb +++ /dev/null @@ -1,23 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisQueues < ::Gitlab::RedisWrapper - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_QUEUES_URL = 'redis://localhost:6381'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_QUEUES_CONFIG_FILE'] || File.expand_path('../../config/redis.queues.yml', __dir__) - end - - def default_url - DEFAULT_QUEUES_URL - end - end -end diff --git a/lib/gitlab/redis_shared_state.rb b/lib/gitlab/redis_shared_state.rb deleted file mode 100644 index cba6c30d5024..000000000000 --- a/lib/gitlab/redis_shared_state.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require_relative 'redis_wrapper' - -module Gitlab - class RedisSharedState < ::Gitlab::RedisWrapper - SESSION_NAMESPACE = 'session:gitlab'.freeze - DEFAULT_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - - class << self - delegate :_raw_config, :fetch_config, to: :new - end - - def config_file - ENV['GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'] || File.expand_path('../../config/redis.shared_state.yml', __dir__) - end - - def default_url - DEFAULT_SHARED_STATE_URL - end - end -end diff --git a/lib/gitlab/redis_wrapper.rb b/lib/gitlab/redis_wrapper.rb deleted file mode 100644 index 63d247c782d5..000000000000 --- a/lib/gitlab/redis_wrapper.rb +++ /dev/null @@ -1,97 +0,0 @@ -# This file should only be used by sub-classes, not directly by any clients of the sub-classes -# please require all dependencies below: -require 'rails' -require 'redis' -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class RedisWrapper - BaseError = ::Redis::BaseError.freeze - CannotConnectError = ::Redis::CannotConnectError.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - # heuristic constant 5 should be a config setting somewhere -- related to CPU count? - size = 5 - if Sidekiq.server? - # the pool will be used in a multi-threaded context - size += Sidekiq.options[:concurrency] - end - size - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: default_url } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index be5c8a707b8b..cec80becfb95 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 2a30bd6a9e70..80bd9ca757fd 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,7 +174,7 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| result = redis_queues.set(key, value, ex: expire, nx: !overwrite) if result redis_queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 5f2b6a2cdd7c..6416b45d5ef7 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,17 +1,17 @@ namespace :cache do namespace :clear do - # @@ TODO - push implementation into Gitlab::RedisCaching + # @@ TODO - push implementation into Gitlab::Redis::Caching KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear key/value cache" task gitlab_kv_cache: :environment do - Gitlab::RedisCaching.with do |kv_cache| + Gitlab::Redis::Caching.with do |kv_cache| cursor = KV_CACHE_SCAN_START_STOP loop do cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::RedisCaching::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Caching::CACHE_NAMESPACE}*", count: KV_CACHE_CLEAR_BATCH_SIZE ) diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index e96a9d0e43b2..a31e44fa9285 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -41,7 +41,7 @@ let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - let(:gitlab_redis_queues) { Gitlab::RedisQueues.new(Rails.env) } + let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) } it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) @@ -74,7 +74,7 @@ end def clear_queues_raw_config - Gitlab::RedisQueues.remove_instance_variable(:@_raw_config) + Gitlab::Redis::Queues.remove_instance_variable(:@_raw_config) rescue NameError # raised if @_raw_config was not set; ignore end diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index 24d39ee64685..13cb333be2c5 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,9 +19,9 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_caching_check']['status']).to eq('ok') - expect(json_response['redis_queues_check']['status']).to eq('ok') - expect(json_response['redis_shared_state_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') end @@ -44,9 +44,9 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_caching_check']['status']).to eq('ok') - expect(json_response['redis_queues_check']['status']).to eq('ok') - expect(json_response['redis_shared_state_check']['status']).to eq('ok') + expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') end end diff --git a/spec/fixtures/config/redis_config_with_env.yml b/spec/fixtures/config/redis_config_with_env.yml new file mode 100644 index 000000000000..f5860f37e475 --- /dev/null +++ b/spec/fixtures/config/redis_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_URL'] %> diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/redis_new_format_host.yml new file mode 100644 index 000000000000..8d134d467e9b --- /dev/null +++ b/spec/fixtures/config/redis_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: slave1 + port: 26379 # point to sentinel, not to redis port + - + host: slave2 + port: 26379 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_new_format_socket.yml b/spec/fixtures/config/redis_new_format_socket.yml new file mode 100644 index 000000000000..4e76830c281d --- /dev/null +++ b/spec/fixtures/config/redis_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.sock +test: + url: unix:/path/to/redis.sock +production: + url: unix:/path/to/redis.sock diff --git a/spec/fixtures/config/redis_old_format_host.yml b/spec/fixtures/config/redis_old_format_host.yml new file mode 100644 index 000000000000..253d0a994f5e --- /dev/null +++ b/spec/fixtures/config/redis_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6379/99 +test: redis://:mypassword@localhost:6379/99 +production: redis://:mypassword@localhost:6379/99 diff --git a/spec/fixtures/config/redis_old_format_socket.yml b/spec/fixtures/config/redis_old_format_socket.yml new file mode 100644 index 000000000000..fd31ce8ea3d7 --- /dev/null +++ b/spec/fixtures/config/redis_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.sock +test: unix:/path/to/old/redis.sock +production: unix:/path/to/old/redis.sock diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index 74667a5a1ad4..e6bd6b276fe7 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -50,7 +50,7 @@ it 'only connects to redis_caching twice' do # Once to load, once to store in the cache - expect(Gitlab::RedisCaching).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Redis::Caching).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -60,7 +60,7 @@ describe 'when a status was cached in redis_caching' do before do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -77,7 +77,7 @@ end it 'only connects to redis_caching once' do - expect(Gitlab::RedisCaching).to receive(:with).exactly(1).and_call_original + expect(Gitlab::Redis::Caching).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -95,7 +95,7 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -209,7 +209,7 @@ pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') @@ -222,7 +222,7 @@ pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil @@ -236,7 +236,7 @@ ) other_status.store_in_cache_if_needed - sha, status = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,7 +244,7 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::RedisCaching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -269,7 +269,7 @@ let(:ref) { 'master' } before do - Gitlab::RedisCaching.with do |redis_caching| + Gitlab::Redis::Caching.with do |redis_caching| redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -295,7 +295,7 @@ it 'deletes values from redis_caching' do pipeline_status.delete_from_cache - key_exists = Gitlab::RedisCaching.with { |redis_caching| redis_caching.exists(cache_key) } + key_exists = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index bd1efdf20a3e..a77a24de5aef 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::RedisCaching::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Caching::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb similarity index 52% rename from spec/lib/gitlab/health_checks/redis_caching_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/caching_check_spec.rb index 710028a4ec8d..0e66bd04303f 100644 --- a/spec/lib/gitlab/health_checks/redis_caching_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisCachingCheck do +describe Gitlab::HealthChecks::Redis::CachingCheck do include_examples 'simple_check', 'redis_caching_ping', 'RedisCaching', 'PONG' end diff --git a/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb similarity index 52% rename from spec/lib/gitlab/health_checks/redis_queues_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/queues_check_spec.rb index 52727f5a8b34..c69443d205d0 100644 --- a/spec/lib/gitlab/health_checks/redis_queues_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisQueuesCheck do +describe Gitlab::HealthChecks::Redis::QueuesCheck do include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG' end diff --git a/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb similarity index 53% rename from spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb rename to spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb index 8b34a9e07083..b72e152bbe2f 100644 --- a/spec/lib/gitlab/health_checks/redis_shared_state_check_spec.rb +++ b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -require_relative './simple_check_shared' +require_relative '../simple_check_shared' -describe Gitlab::HealthChecks::RedisSharedStateCheck do +describe Gitlab::HealthChecks::Redis::SharedStateCheck do include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG' end diff --git a/spec/lib/gitlab/redis_caching_spec.rb b/spec/lib/gitlab/redis/caching_spec.rb similarity index 98% rename from spec/lib/gitlab/redis_caching_spec.rb rename to spec/lib/gitlab/redis/caching_spec.rb index f373716a0b39..1ff8dc1e78a8 100644 --- a/spec/lib/gitlab/redis_caching_spec.rb +++ b/spec/lib/gitlab/redis/caching_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisCaching do +describe Gitlab::Redis::Caching do include StubENV let(:config) { 'config/redis.caching.yml' } @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisCaching::DEFAULT_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_REDIS_CACHING_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis_queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb similarity index 98% rename from spec/lib/gitlab/redis_queues_spec.rb rename to spec/lib/gitlab/redis/queues_spec.rb index b498f48cd236..d4878349edd1 100644 --- a/spec/lib/gitlab/redis_queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisQueues do +describe Gitlab::Redis::Queues do include StubENV let(:config) { 'config/redis.queues.yml' } @@ -180,7 +180,7 @@ it 'returns default queues url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisQueues::DEFAULT_QUEUES_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis_shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb similarity index 98% rename from spec/lib/gitlab/redis_shared_state_spec.rb rename to spec/lib/gitlab/redis/shared_state_spec.rb index 449691980d46..978b6c6096ef 100644 --- a/spec/lib/gitlab/redis_shared_state_spec.rb +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::RedisSharedState do +describe Gitlab::Redis::SharedState do include StubENV let(:config) { 'config/redis.shared_state.yml' } @@ -180,7 +180,7 @@ it 'returns default redis_shared_state url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::RedisSharedState::DEFAULT_SHARED_STATE_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb new file mode 100644 index 000000000000..214361c6ece9 --- /dev/null +++ b/spec/lib/gitlab/redis/wrapper_spec.rb @@ -0,0 +1,211 @@ +require 'spec_helper' + +describe Gitlab::Redis::Wrapper do + include StubENV + + let(:config) { 'config/resque.yml' } + + before(:each) do + stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s) + clear_raw_config + end + + after(:each) do + clear_raw_config + end + + describe '.params' do + subject { described_class.params } + + it 'withstands mutation' do + params1 = described_class.params + params2 = described_class.params + params1[:foo] = :bar + + expect(params2).not_to have_key(:foo) + end + + context 'when url contains unix socket reference' do + let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/old/redis.sock') + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns path key instead' do + is_expected.to include(path: '/path/to/redis.sock') + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' } + + context 'with old format' do + let(:config) { config_old } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config) { config_new } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' + + expect(url2).not_to end_with('foobar') + end + + context 'when yml file with env variable' do + let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' } + + before do + stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379') + end + + it 'reads redis url from env variable' do + expect(described_class.url).to eq 'redis://redishost:6379' + end + end + end + + describe '._raw_config' do + subject { described_class._raw_config } + let(:config) { '/var/empty/doesnotexist' } + + it 'should be frozen' do + expect(subject).to be_frozen + end + + it 'returns false when the file does not exist' do + expect(subject).to eq(false) + end + end + + describe '.with' do + before { clear_pool } + after { clear_pool } + + context 'when running not on sidekiq workers' do + before { allow(Sidekiq).to receive(:server?).and_return(false) } + + it 'instantiates a connection pool with size 5' do + expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original + + described_class.with { |_wrapper| true } + end + end + + context 'when running on sidekiq workers' do + before do + allow(Sidekiq).to receive(:server?).and_return(true) + allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) + end + + it 'instantiates a connection pool with a size based on the concurrency of the worker' do + expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original + + described_class.with { |_wrapper| true } + end + end + end + + describe '#sentinels' do + subject { described_class.new(Rails.env).sentinels } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + + it 'returns an array of hashes with host and port keys' do + is_expected.to include(host: 'localhost', port: 26379) + is_expected.to include(host: 'slave2', port: 26379) + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#sentinels?' do + subject { described_class.new(Rails.env).sentinels? } + + context 'when sentinels are defined' do + let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + + it 'returns true' do + is_expected.to be_truthy + end + end + + context 'when sentinels are not defined' do + let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + + it 'returns false' do + is_expected.to be_falsey + end + end + end + + describe '#raw_config_hash' do + it 'returns default redis url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') + end + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(described_class).to receive(:_raw_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end + + def clear_raw_config + described_class.remove_instance_variable(:@_raw_config) + rescue NameError + # raised if @_raw_config was not set; ignore + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end +end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index b8190deae995..9a9011044590 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -11,7 +11,7 @@ described_class.record(42) end - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end @@ -21,7 +21,7 @@ it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end @@ -34,13 +34,13 @@ it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -48,13 +48,13 @@ context 'and key does not exist' do it 'removes the pair from SharedState' do - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -67,13 +67,13 @@ described_class.record(41, now) described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -83,13 +83,13 @@ it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) end end diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index b5cebd55696e..d30d7fd92ddb 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -9,7 +9,7 @@ let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::RedisSharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis_shared_state| redis_shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 5e5a90630559..678c02c851fa 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -311,7 +311,7 @@ end def expect_value_in_queues - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| runner_queue_key = runner.send(:runner_queue_key) expect(redis_queues.get(runner_queue_key)) end @@ -330,7 +330,7 @@ def expect_value_in_queues end it 'cleans up the queue' do - Gitlab::RedisQueues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis_queues| expect(redis_queues.get(queue_key)).to be_nil end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index c3d04edb7e6d..c12e276c9a70 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -526,17 +526,17 @@ before do # Flush any raw key-value data stored by the housekeeping code. - Gitlab::RedisCaching.with { |conn| conn.flushall } - Gitlab::RedisQueues.with { |conn| conn.flushall } - Gitlab::RedisSharedState.with { |conn| conn.flushall } + Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping) end after do - Gitlab::RedisCaching.with { |conn| conn.flushall } - Gitlab::RedisQueues.with { |conn| conn.flushall } - Gitlab::RedisSharedState.with { |conn| conn.flushall } + Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } end it 'does not perform housekeeping when not needed' do diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 7cae0f569265..114357ff5c77 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::RedisSharedState::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::Redis::SharedState::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Gitlab::RedisSharedState::CannotConnectError + rescue Gitlab::Redis::SharedState::CannotConnectError end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 722b1c3c0c6b..741aacc0dad7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,4 @@ require './spec/simplecov_env' -require './lib/gitlab/redis_caching' -require './lib/gitlab/redis_queues' -require './lib/gitlab/redis_shared_state' SimpleCovEnv.start! ENV["RAILS_ENV"] ||= 'test' @@ -89,20 +86,20 @@ end config.around(:each, :clean_gitlab_redis_caching) do |example| - Gitlab::RedisCaching.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) example.run - Gitlab::RedisCaching.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) end config.around(:each, :clean_gitlab_redis_shared_state) do |example| - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) example.run - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) end diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 3a72b2c41a71..6e25c5361916 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,9 +1,9 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::RedisCaching.with(&:flushall) - Gitlab::RedisQueues.with(&:flushall) - Gitlab::RedisSharedState.with(&:flushall) + Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Queues.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) end before do -- GitLab From 9cb9d0457d94238776865ebdf40e2791e975679f Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 30 May 2017 18:06:48 +0800 Subject: [PATCH 17/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 --- lib/gitlab/redis/wrapper.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 78f00d26585d..011708b8b26f 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -106,17 +106,17 @@ def config_file proposed_file_name = ENV[@environment_var_name] file_name = proposed_file_name if proposed_file_name # otherwise, if config files exists for sub-class, use it - if !file_name + unless file_name proposed_file_name = File.expand_path("../../../config/#{@config_file_name}", __dir__) file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) end # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - if !file_name + unless file_name proposed_file_name = ENV[ENV_VAR_NAME] file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) end # otherwise, if config files exists for wrapper class, use it - if !file_name + unless file_name proposed_file_name = File.expand_path("../../../config/#{CONFIG_FILE_NAME}", __dir__) file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) end -- GitLab From 2bae1e9d5b8a22e87bc326069ffa78645e8fbab4 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 10:04:42 +0800 Subject: [PATCH 18/62] defer locale issue --- .gitignore | 1 - locale/gitlab.pot | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index bf7dadf5cc3f..f37ce6836e31 100644 --- a/.gitignore +++ b/.gitignore @@ -61,5 +61,4 @@ eslint-report.html /shared/* /.gitlab_workhorse_secret /webpack-report/ -/locale/gitlab.pot /locale/**/LC_MESSAGES diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5a2aff5fffc9..3967d40ea9e5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8,8 +8,8 @@ msgid "" msgstr "" "Project-Id-Version: gitlab 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-05-25 12:34+0800\n" -"PO-Revision-Date: 2017-05-25 12:34+0800\n" +"POT-Creation-Date: 2017-05-04 19:24-0500\n" +"PO-Revision-Date: 2017-05-04 19:24-0500\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" -- GitLab From 3cf9a69e2c80f54b70e2fbb8e7725caf2d03562e Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 10:08:08 +0800 Subject: [PATCH 19/62] deferred to later issue --- .gitlab-ci.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc8d0dba05e4..dea11bb9f61e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,39 +54,11 @@ stages: services: - postgres:9.2 - redis:alpine -# -# notes for when CI supports service configuration on shared image -# - caching: -# image: redis:alpine -# ports: -# - "6380:6379" -# - queues: -# image: redis:alpine -# ports: -# - "6381:6379" -# - redis_shared_state: -# image: redis:alpine -# ports: -# - "6382:6379" .use-mysql: &use-mysql services: - mysql:latest - redis:alpine -# -# notes for when CI supports service configuration on shared image -# - caching: -# image: redis:alpine -# ports: -# - "6380:6379" -# - queues: -# image: redis:alpine -# ports: -# - "6381:6379" -# - redis_shared_state: -# image: redis:alpine -# ports: -# - "6382:6379" .only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql only: -- GitLab From 54dd174a06fa98eb3d4c6f551096eb3367de7082 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 10:15:55 +0800 Subject: [PATCH 20/62] restore original contents of `config/resque.yml` --- config/resque.yml.example | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/config/resque.yml.example b/config/resque.yml.example index e983be5b32e4..0c19d8bc1d36 100644 --- a/config/resque.yml.example +++ b/config/resque.yml.example @@ -3,24 +3,18 @@ # development: url: redis://localhost:6379 - # - # notes for when CI supports service configuration on shared image - # url: redis://localhost:6379 # sentinels: # - # host: localhost - # port: 26379 # point to sentinel, not to redis port + # port: 26380 # point to sentinel, not to redis port # - # host: slave2 - # port: 26379 # point to sentinel, not to redis port + # port: 26381 # point to sentinel, not to redis port test: url: redis://localhost:6379 - # - # notes for when CI supports service configuration on shared image - # url: redis://localhost:6379 production: # Redis (single instance) - url: unix:/var/run/redis/redis.caching.sock + url: unix:/var/run/redis/redis.sock ## # Redis + Sentinel (for HA) # -- GitLab From bd916fba0400875fe82c9bed0412750e0df0a5d3 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 10:35:44 +0800 Subject: [PATCH 21/62] deferred to https://gitlab.com/gitlab-org/gitlab-ce/issues/33112 --- lib/gitlab/mail_room.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index a2b363e54085..9f432673a6ec 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -37,7 +37,6 @@ def fetch_config gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) config[:redis_url] = gitlab_redis_queues.url - # @@ TODO this is too-much-information on topology in the client of an object if gitlab_redis_queues.sentinels? config[:sentinels] = gitlab_redis_queues.sentinels end -- GitLab From 2418e768af587045288de0ce9f5c9e84ea98dc95 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 11:07:49 +0800 Subject: [PATCH 22/62] increase number of knapsack containers --- .gitlab-ci.yml | 80 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dea11bb9f61e..6e861d9d822d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,26 +202,66 @@ setup-test-env: - public/assets - tmp/tests -rspec-pg 0 20: *rspec-knapsack-pg -rspec-pg 1 20: *rspec-knapsack-pg -rspec-pg 2 20: *rspec-knapsack-pg -rspec-pg 3 20: *rspec-knapsack-pg -rspec-pg 4 20: *rspec-knapsack-pg -rspec-pg 5 20: *rspec-knapsack-pg -rspec-pg 6 20: *rspec-knapsack-pg -rspec-pg 7 20: *rspec-knapsack-pg -rspec-pg 8 20: *rspec-knapsack-pg -rspec-pg 9 20: *rspec-knapsack-pg -rspec-pg 10 20: *rspec-knapsack-pg -rspec-pg 11 20: *rspec-knapsack-pg -rspec-pg 12 20: *rspec-knapsack-pg -rspec-pg 13 20: *rspec-knapsack-pg -rspec-pg 14 20: *rspec-knapsack-pg -rspec-pg 15 20: *rspec-knapsack-pg -rspec-pg 16 20: *rspec-knapsack-pg -rspec-pg 17 20: *rspec-knapsack-pg -rspec-pg 18 20: *rspec-knapsack-pg -rspec-pg 19 20: *rspec-knapsack-pg +rspec-pg 0 60: *rspec-knapsack-pg +rspec-pg 1 60: *rspec-knapsack-pg +rspec-pg 2 60: *rspec-knapsack-pg +rspec-pg 3 60: *rspec-knapsack-pg +rspec-pg 4 60: *rspec-knapsack-pg +rspec-pg 5 60: *rspec-knapsack-pg +rspec-pg 6 60: *rspec-knapsack-pg +rspec-pg 7 60: *rspec-knapsack-pg +rspec-pg 8 60: *rspec-knapsack-pg +rspec-pg 9 60: *rspec-knapsack-pg +rspec-pg 10 60: *rspec-knapsack-pg +rspec-pg 11 60: *rspec-knapsack-pg +rspec-pg 12 60: *rspec-knapsack-pg +rspec-pg 13 60: *rspec-knapsack-pg +rspec-pg 14 60: *rspec-knapsack-pg +rspec-pg 15 60: *rspec-knapsack-pg +rspec-pg 16 60: *rspec-knapsack-pg +rspec-pg 17 60: *rspec-knapsack-pg +rspec-pg 18 60: *rspec-knapsack-pg +rspec-pg 19 60: *rspec-knapsack-pg +rspec-pg 20 60: *rspec-knapsack-pg +rspec-pg 21 60: *rspec-knapsack-pg +rspec-pg 22 60: *rspec-knapsack-pg +rspec-pg 23 60: *rspec-knapsack-pg +rspec-pg 24 60: *rspec-knapsack-pg +rspec-pg 25 60: *rspec-knapsack-pg +rspec-pg 26 60: *rspec-knapsack-pg +rspec-pg 27 60: *rspec-knapsack-pg +rspec-pg 28 60: *rspec-knapsack-pg +rspec-pg 29 60: *rspec-knapsack-pg +rspec-pg 30 60: *rspec-knapsack-pg +rspec-pg 31 60: *rspec-knapsack-pg +rspec-pg 32 60: *rspec-knapsack-pg +rspec-pg 33 60: *rspec-knapsack-pg +rspec-pg 34 60: *rspec-knapsack-pg +rspec-pg 35 60: *rspec-knapsack-pg +rspec-pg 36 60: *rspec-knapsack-pg +rspec-pg 37 60: *rspec-knapsack-pg +rspec-pg 38 60: *rspec-knapsack-pg +rspec-pg 39 60: *rspec-knapsack-pg +rspec-pg 40 60: *rspec-knapsack-pg +rspec-pg 41 60: *rspec-knapsack-pg +rspec-pg 42 60: *rspec-knapsack-pg +rspec-pg 43 60: *rspec-knapsack-pg +rspec-pg 44 60: *rspec-knapsack-pg +rspec-pg 45 60: *rspec-knapsack-pg +rspec-pg 46 60: *rspec-knapsack-pg +rspec-pg 47 60: *rspec-knapsack-pg +rspec-pg 48 60: *rspec-knapsack-pg +rspec-pg 49 60: *rspec-knapsack-pg +rspec-pg 50 60: *rspec-knapsack-pg +rspec-pg 51 60: *rspec-knapsack-pg +rspec-pg 52 60: *rspec-knapsack-pg +rspec-pg 53 60: *rspec-knapsack-pg +rspec-pg 54 60: *rspec-knapsack-pg +rspec-pg 55 60: *rspec-knapsack-pg +rspec-pg 56 60: *rspec-knapsack-pg +rspec-pg 57 60: *rspec-knapsack-pg +rspec-pg 58 60: *rspec-knapsack-pg +rspec-pg 59 60: *rspec-knapsack-pg rspec-mysql 0 20: *rspec-knapsack-mysql rspec-mysql 1 20: *rspec-knapsack-mysql -- GitLab From 5291f9fca9c978d720d468c58ca86dc9949b3210 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 11:30:18 +0800 Subject: [PATCH 23/62] revert name of block variable to "redis" --- app/models/ci/runner.rb | 4 +- app/models/project.rb | 6 +-- lib/gitlab/auth/unique_ips_limiter.rb | 4 +- .../cache/ci/project_pipeline_status.rb | 24 +++++------ lib/gitlab/chat_name_token.rb | 12 +++--- lib/gitlab/etag_caching/store.rb | 6 +-- lib/gitlab/exclusive_lease.rb | 12 +++--- lib/gitlab/lfs_token.rb | 6 +-- lib/gitlab/user_activities.rb | 12 +++--- lib/gitlab/workhorse.rb | 8 ++-- .../cache/ci/project_pipeline_status_spec.rb | 26 ++++++------ spec/lib/gitlab/user_activities_spec.rb | 40 +++++++++---------- ...tivities_to_users_last_activity_on_spec.rb | 4 +- spec/models/ci/runner_spec.rb | 8 ++-- 14 files changed, 86 insertions(+), 86 deletions(-) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d2be57e36ff0..05bca7fea10a 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,8 +145,8 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::Redis::Queues.with do |redis_queues| - redis_queues.del(runner_queue_key) + Gitlab::Redis::Queues.with do |redis| + redis.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index e865409e6f2c..60dee71d4038 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1279,15 +1279,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } + Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 6fff4309e893..baa1f802d8a0 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,9 +27,9 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis::SharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil - redis_shared_state.multi do |r| + redis.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) unique_ips_count = r.zcard(key) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 492e1254f164..a60ba0f683fa 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,12 +23,12 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.multi do + result = Gitlab::Redis::Caching.with do |redis| + redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) - redis_caching.exists(cache_key) - redis_caching.hmget(cache_key, :sha, :status, :ref) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) end end end @@ -100,28 +100,28 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::Redis::Caching.with do |redis_caching| - self.sha, self.status, self.ref = redis_caching.hmget(cache_key, :sha, :status, :ref) + Gitlab::Redis::Caching.with do |redis| + self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::Redis::Caching.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.del(cache_key) + Gitlab::Redis::Caching.with do |redis| + redis.del(cache_key) end end def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.exists(cache_key) + Gitlab::Redis::Caching.with do |redis| + redis.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index bf8d442140fc..e63e54373313 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ def initialize(token = new_token) end def get - Gitlab::Redis::SharedState.with do |redis_shared_state| - data = redis_shared_state.get(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis::SharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis_shared_state.set(redis_shared_state_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.del(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 7096be42beab..309a542595fb 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -5,14 +5,14 @@ class Store SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 19f3b7b50b74..c60ca532334a 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,8 +18,8 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end @@ -37,15 +37,15 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.exists(@redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 043bfe18545b..8e57ba831c51 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ def initialize(actor) end def token - Gitlab::Redis::SharedState.with do |redis_shared_state| - token = redis_shared_state.get(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis_shared_state.set(redis_shared_state_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index cec80becfb95..125488536e19 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,14 +6,14 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hset(KEY, key, time.to_i) + Gitlab::Redis::SharedState.with do |redis| + redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hdel(KEY, keys) + Gitlab::Redis::SharedState.with do |redis| + redis.hdel(KEY, keys) end end @@ -21,8 +21,8 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hscan(KEY, cursor, count: BATCH_SIZE) + Gitlab::Redis::SharedState.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) end Hash[pairs].each { |pair| yield pair } diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 80bd9ca757fd..256836d9716b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,13 +174,13 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis_queues| - result = redis_queues.set(key, value, ex: expire, nx: !overwrite) + Gitlab::Redis::Queues.with do |redis| + result = redis.set(key, value, ex: expire, nx: !overwrite) if result - redis_queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") value else - redis_queues.get(key) + redis.get(key) end end end diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index e6bd6b276fe7..69afbff2ad9a 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -60,8 +60,8 @@ describe 'when a status was cached in redis_caching' do before do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Caching.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end @@ -95,8 +95,8 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::Redis::Caching.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end result = [{ loaded_from_cache: false, pipeline_info: { sha: nil, status: nil, ref: nil } }, @@ -209,7 +209,7 @@ pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') @@ -222,7 +222,7 @@ pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil @@ -236,7 +236,7 @@ ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,8 +244,8 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Caching.with do |redis| + redis.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Caching.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -269,8 +269,8 @@ let(:ref) { 'master' } before do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Caching.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end @@ -295,7 +295,7 @@ it 'deletes values from redis_caching' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.exists(cache_key) } + key_exists = Gitlab::Redis::Caching.with { |redis| redis.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index 9a9011044590..a4ea0ac59e96 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -11,8 +11,8 @@ described_class.record(42) end - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -21,8 +21,8 @@ it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -34,28 +34,28 @@ it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and key does not exist' do it 'removes the pair from SharedState' do - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -67,14 +67,14 @@ described_class.record(41, now) described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -83,14 +83,14 @@ it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index d30d7fd92ddb..a7be02f24278 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -9,8 +9,8 @@ let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) + Gitlab::Redis::SharedState.with do |redis| + redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 678c02c851fa..4b9cce28e0e6 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -311,9 +311,9 @@ end def expect_value_in_queues - Gitlab::Redis::Queues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis| runner_queue_key = runner.send(:runner_queue_key) - expect(redis_queues.get(runner_queue_key)) + expect(redis.get(runner_queue_key)) end end end @@ -330,8 +330,8 @@ def expect_value_in_queues end it 'cleans up the queue' do - Gitlab::Redis::Queues.with do |redis_queues| - expect(redis_queues.get(queue_key)).to be_nil + Gitlab::Redis::Queues.with do |redis| + expect(redis.get(queue_key)).to be_nil end end end -- GitLab From 1b58cf0fabe6a4212622f5950ecd2e23775eb751 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 13:28:27 +0800 Subject: [PATCH 24/62] rename redis_caching to redis_cache --- .gitignore | 2 +- app/controllers/health_controller.rb | 2 +- config/application.rb | 6 +-- config/initializers/7_redis.rb | 2 +- ...ng.yml.example => redis.cache.yml.example} | 4 +- config/redis.queues.yml.example | 2 - config/redis.shared_state.yml.example | 2 - .../cache/ci/project_pipeline_status.rb | 10 ++-- lib/gitlab/current_settings.rb | 2 +- .../{caching_check.rb => cache_check.rb} | 6 +-- lib/gitlab/redis/{caching.rb => cache.rb} | 10 ++-- lib/tasks/cache.rake | 6 +-- scripts/prepare_build.sh | 6 +-- spec/controllers/health_controller_spec.rb | 6 +-- .../cache/ci/project_pipeline_status_spec.rb | 48 +++++++++---------- spec/lib/gitlab/current_settings_spec.rb | 2 +- spec/services/git_push_service_spec.rb | 4 +- spec/spec_helper.rb | 6 +-- .../unique_ip_check_shared_examples.rb | 2 +- 19 files changed, 61 insertions(+), 67 deletions(-) rename config/{redis.caching.yml.example => redis.cache.yml.example} (85%) rename lib/gitlab/health_checks/redis/{caching_check.rb => cache_check.rb} (78%) rename lib/gitlab/redis/{caching.rb => cache.rb} (59%) diff --git a/.gitignore b/.gitignore index f37ce6836e31..0490646183e0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ eslint-report.html /config/initializers/smtp_settings.rb /config/initializers/relative_url.rb /config/resque.yml -/config/redis.caching.yml +/config/redis.cache.yml /config/redis.queues.yml /config/redis.shared_state.yml /config/unicorn.rb diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index c1421de14fda..3d061af80540 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,7 +4,7 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::Redis::CachingCheck, + Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck diff --git a/config/application.rb b/config/application.rb index af78c06e3219..52799069673b 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,7 +6,7 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis/caching') + require_dependency Rails.root.join('lib/gitlab/redis/cache') require_dependency Rails.root.join('lib/gitlab/redis/queues') require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') @@ -141,8 +141,8 @@ class Application < Rails::Application end # Use caching across all environments - caching_config_hash = Gitlab::Redis::Caching.params - caching_config_hash[:namespace] = Gitlab::Redis::Caching::CACHE_NAMESPACE + caching_config_hash = Gitlab::Redis::Cache.params + caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 7ad600aa26ae..95ab07e34fad 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::Redis::Caching.with { nil } +Gitlab::Redis::Cache.with { nil } Gitlab::Redis::Queues.with { nil } Gitlab::Redis::SharedState.with { nil } diff --git a/config/redis.caching.yml.example b/config/redis.cache.yml.example similarity index 85% rename from config/redis.caching.yml.example rename to config/redis.cache.yml.example index 8279ddd0c8cd..27478f0a93e4 100644 --- a/config/redis.caching.yml.example +++ b/config/redis.cache.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/10 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6380 # sentinels: # - @@ -16,11 +15,10 @@ development: test: url: redis://localhost:6379/10 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6380 production: # Redis (single instance) - url: unix:/var/run/redis/redis.caching.sock + url: unix:/var/run/redis/redis.cache.sock ## # Redis + Sentinel (for HA) # diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example index 3c954c13fe76..dab1f26b0964 100644 --- a/config/redis.queues.yml.example +++ b/config/redis.queues.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/11 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6381 # sentinels: # - @@ -16,7 +15,6 @@ development: test: url: redis://localhost:6379/11 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6381 production: # Redis (single instance) diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example index 21bc791024fd..9371e3619b78 100644 --- a/config/redis.shared_state.yml.example +++ b/config/redis.shared_state.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/12 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6382 # sentinels: # - @@ -16,7 +15,6 @@ development: test: url: redis://localhost:6379/12 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6382 production: # Redis (single instance) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index a60ba0f683fa..05df66c1a43f 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::Redis::Caching.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ def delete_from_cache def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 88fd408ef475..f659a975022e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::Redis::Caching::BaseError, ::Errno::ENOENT + rescue ::Gitlab::Redis::Cache::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/health_checks/redis/caching_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb similarity index 78% rename from lib/gitlab/health_checks/redis/caching_check.rb rename to lib/gitlab/health_checks/redis/cache_check.rb index a3c1780e64bc..8068276ac5d1 100644 --- a/lib/gitlab/health_checks/redis/caching_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -1,14 +1,14 @@ module Gitlab module HealthChecks module Redis - class CachingCheck + class CacheCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'redis_caching_ping' + 'redis_cache_ping' end def is_successful?(result) @@ -17,7 +17,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::Redis::Caching.with(&:ping) + Gitlab::Redis::Cache.with(&:ping) end end end diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/cache.rb similarity index 59% rename from lib/gitlab/redis/caching.rb rename to lib/gitlab/redis/cache.rb index b9a0149cf880..62032a0e1ca1 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/cache.rb @@ -4,9 +4,9 @@ module Gitlab module Redis - class Caching < ::Gitlab::Redis::Wrapper + class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_REDIS_CACHING_URL = 'redis://localhost:6380'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze class << self delegate :_raw_config, :fetch_config, :config_file, to: :new @@ -14,9 +14,9 @@ class << self def initialize(rails_env = nil) @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_CACHING_URL - @environment_var_name = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze - @config_file_name = 'redis.caching.yml'.freeze + @default_url = DEFAULT_REDIS_CACHE_URL + @environment_var_name = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + @config_file_name = 'redis.cache.yml'.freeze end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 6416b45d5ef7..1b061b5e4ff5 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,17 +1,17 @@ namespace :cache do namespace :clear do - # @@ TODO - push implementation into Gitlab::Redis::Caching + # @@ TODO - push implementation into Gitlab::Redis::Cache KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear key/value cache" task gitlab_kv_cache: :environment do - Gitlab::Redis::Caching.with do |kv_cache| + Gitlab::Redis::Cache.with do |kv_cache| cursor = KV_CACHE_SCAN_START_STOP loop do cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::Redis::Caching::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: KV_CACHE_CLEAR_BATCH_SIZE ) diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index a4e3cefbcaa9..9088a387d6a5 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -34,11 +34,11 @@ else # Assume it's mysql sed -i 's/# host:.*/host: mysql/g' config/database.yml fi -cp config/redis.caching.yml.example config/redis.caching.yml -sed -i 's/localhost/redis/g' config/redis.caching.yml +cp config/redis.cache.yml.example config/redis.cache.yml +sed -i 's/localhost/redis/g' config/redis.cache.yml # # notes for when CI supports service configuration on shared image -#sed -i 's/localhost/caching/g' config/redis.caching.yml +#sed -i 's/localhost/caching/g' config/redis.cache.yml cp config/redis.queues.yml.example config/redis.queues.yml sed -i 's/localhost/redis/g' config/redis.queues.yml diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index 13cb333be2c5..f9e4171791a2 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -74,9 +74,9 @@ it 'returns Caching ping metrics' do get :metrics - expect(response.body).to match(/^redis_caching_ping_timeout 0$/) - expect(response.body).to match(/^redis_caching_ping_success 1$/) - expect(response.body).to match(/^redis_caching_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^redis_cache_ping_timeout 0$/) + expect(response.body).to match(/^redis_cache_ping_success 1$/) + expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/) end it 'returns Queues ping metrics' do diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index 69afbff2ad9a..a6741f16113e 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_caching do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do let!(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } let(:cache_key) { "projects/#{project.id}/pipeline_status" } @@ -28,8 +28,8 @@ expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) end - describe 'without a status in redis_caching' do - it 'loads the status from a commit when it was not in redis_caching' do + describe 'without a status in redis_cache' do + it 'loads the status from a commit when it was not in redis_cache' do empty_status = { sha: nil, status: nil, ref: nil } fake_pipeline = described_class.new( project_without_status, @@ -48,9 +48,9 @@ described_class.load_in_batch_for_projects([project_without_status]) end - it 'only connects to redis_caching twice' do + it 'only connects to redis_cache twice' do # Once to load, once to store in the cache - expect(Gitlab::Redis::Caching).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -58,9 +58,9 @@ end end - describe 'when a status was cached in redis_caching' do + describe 'when a status was cached in redis_cache' do before do - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -76,8 +76,8 @@ expect(pipeline_status.ref).to eq(ref) end - it 'only connects to redis_caching once' do - expect(Gitlab::Redis::Caching).to receive(:with).exactly(1).and_call_original + it 'only connects to redis_cache once' do + expect(Gitlab::Redis::Cache).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -95,7 +95,7 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -183,7 +183,7 @@ end end - describe "#load_from_project", :clean_gitlab_redis_caching do + describe "#load_from_project", :clean_gitlab_redis_cache do let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } it 'reads the status from the pipeline for the commit' do @@ -203,40 +203,40 @@ end end - describe "#store_in_cache", :clean_gitlab_redis_caching do + describe "#store_in_cache", :clean_gitlab_redis_cache do it "sets the object in caching" do pipeline_status.sha = '123456' pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') end end - describe '#store_in_cache_if_needed', :clean_gitlab_redis_caching do + describe '#store_in_cache_if_needed', :clean_gitlab_redis_cache do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil expect(ref).not_to be_nil end - it "doesn't store the status in redis_caching when the sha is not the head of the project" do + it "doesn't store the status in redis_cache when the sha is not the head of the project" do other_status = described_class.new( project, pipeline_info: { sha: "123456", status: "failed" } ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis::Caching.with { |redis| redis.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,7 +244,7 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -263,20 +263,20 @@ end end - describe "with a status in caching", :clean_gitlab_redis_caching do + describe "with a status in caching", :clean_gitlab_redis_cache do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:ref) { 'master' } before do - Gitlab::Redis::Caching.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end describe '#load_from_cache' do - it 'reads the status from redis_caching' do + it 'reads the status from redis_cache' do pipeline_status.load_from_cache expect(pipeline_status.sha).to eq(sha) @@ -292,10 +292,10 @@ end describe '#delete_from_cache' do - it 'deletes values from redis_caching' do + it 'deletes values from redis_cache' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis::Caching.with { |redis| redis.exists(cache_key) } + key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index a77a24de5aef..55fb547b8dd2 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Caching::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Cache::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 12468c4a9097..6091fa6f0bab 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -527,7 +527,7 @@ before do # Flush any raw key-value data stored by the housekeeping code. - Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Cache.with { |conn| conn.flushall } Gitlab::Redis::Queues.with { |conn| conn.flushall } Gitlab::Redis::SharedState.with { |conn| conn.flushall } @@ -535,7 +535,7 @@ end after do - Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Cache.with { |conn| conn.flushall } Gitlab::Redis::Queues.with { |conn| conn.flushall } Gitlab::Redis::SharedState.with { |conn| conn.flushall } end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 741aacc0dad7..61a4fbfaa8d0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -85,12 +85,12 @@ Rails.cache = caching_store end - config.around(:each, :clean_gitlab_redis_caching) do |example| - Gitlab::Redis::Caching.with(&:flushall) + config.around(:each, :clean_gitlab_redis_cache) do |example| + Gitlab::Redis::Cache.with(&:flushall) example.run - Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Cache.with(&:flushall) end config.around(:each, :clean_gitlab_redis_shared_state) do |example| diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 6e25c5361916..ee2b7de925b2 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,7 +1,7 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Cache.with(&:flushall) Gitlab::Redis::Queues.with(&:flushall) Gitlab::Redis::SharedState.with(&:flushall) end -- GitLab From 6d71d3af07cdb64e9d97e0a0e3ec570ac4d2b75c Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 13:30:48 +0800 Subject: [PATCH 25/62] rename redis_caching to redis_cache --- ...st.yml => redis_cache_new_format_host.yml} | 0 ...st.yml => redis_cache_old_format_host.yml} | 0 .../redis/{caching_spec.rb => cache_spec.rb} | 32 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) rename spec/fixtures/config/{redis_caching_new_format_host.yml => redis_cache_new_format_host.yml} (100%) rename spec/fixtures/config/{redis_caching_old_format_host.yml => redis_cache_old_format_host.yml} (100%) rename spec/lib/gitlab/redis/{caching_spec.rb => cache_spec.rb} (80%) diff --git a/spec/fixtures/config/redis_caching_new_format_host.yml b/spec/fixtures/config/redis_cache_new_format_host.yml similarity index 100% rename from spec/fixtures/config/redis_caching_new_format_host.yml rename to spec/fixtures/config/redis_cache_new_format_host.yml diff --git a/spec/fixtures/config/redis_caching_old_format_host.yml b/spec/fixtures/config/redis_cache_old_format_host.yml similarity index 100% rename from spec/fixtures/config/redis_caching_old_format_host.yml rename to spec/fixtures/config/redis_cache_old_format_host.yml diff --git a/spec/lib/gitlab/redis/caching_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb similarity index 80% rename from spec/lib/gitlab/redis/caching_spec.rb rename to spec/lib/gitlab/redis/cache_spec.rb index 1ff8dc1e78a8..3f36e2164b5e 100644 --- a/spec/lib/gitlab/redis/caching_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::Redis::Caching do +describe Gitlab::Redis::Cache do include StubENV - let(:config) { 'config/redis.caching.yml' } + let(:config) { 'config/redis.cache.yml' } before(:each) do - stub_env('GITLAB_REDIS_CACHING_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_REDIS_CACHE_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.caching.sock') + is_expected.to include(path: '/path/to/old/redis.cache.sock') is_expected.not_to have_key(:url) end end @@ -42,15 +42,15 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.caching.sock') + is_expected.to include(path: '/path/to/redis.cache.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } context 'with old format' do let(:config) { config_old } @@ -82,10 +82,10 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_caching_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_config_with_env.yml' } before do - stub_env('TEST_GITLAB_REDIS_CACHING_URL', 'redis://redishost:6380') + stub_env('TEST_GITLAB_REDIS_CACHE_URL', 'redis://redishost:6380') end it 'reads caching url from env variable' do @@ -139,7 +139,7 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do is_expected.to include(host: 'localhost', port: 26380) @@ -148,7 +148,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_REDIS_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL) end it 'returns old-style single url config in a hash' do -- GitLab From e404284e873969fb2eb1721b266daf30dbc02405 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 13:31:45 +0800 Subject: [PATCH 26/62] rename redis_caching to redis_cache --- spec/fixtures/config/redis_cache_config_with_env.yml | 2 ++ spec/fixtures/config/redis_cache_new_format_socket.yml | 6 ++++++ spec/fixtures/config/redis_cache_old_format_socket.yml | 3 +++ spec/fixtures/config/redis_caching_config_with_env.yml | 2 -- spec/fixtures/config/redis_caching_new_format_socket.yml | 6 ------ spec/fixtures/config/redis_caching_old_format_socket.yml | 3 --- spec/lib/gitlab/health_checks/redis/cache_check_spec.rb | 6 ++++++ spec/lib/gitlab/health_checks/redis/caching_check_spec.rb | 6 ------ 8 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 spec/fixtures/config/redis_cache_config_with_env.yml create mode 100644 spec/fixtures/config/redis_cache_new_format_socket.yml create mode 100644 spec/fixtures/config/redis_cache_old_format_socket.yml delete mode 100644 spec/fixtures/config/redis_caching_config_with_env.yml delete mode 100644 spec/fixtures/config/redis_caching_new_format_socket.yml delete mode 100644 spec/fixtures/config/redis_caching_old_format_socket.yml create mode 100644 spec/lib/gitlab/health_checks/redis/cache_check_spec.rb delete mode 100644 spec/lib/gitlab/health_checks/redis/caching_check_spec.rb diff --git a/spec/fixtures/config/redis_cache_config_with_env.yml b/spec/fixtures/config/redis_cache_config_with_env.yml new file mode 100644 index 000000000000..52fd5a064605 --- /dev/null +++ b/spec/fixtures/config/redis_cache_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %> diff --git a/spec/fixtures/config/redis_cache_new_format_socket.yml b/spec/fixtures/config/redis_cache_new_format_socket.yml new file mode 100644 index 000000000000..3634c5501633 --- /dev/null +++ b/spec/fixtures/config/redis_cache_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.cache.sock +test: + url: unix:/path/to/redis.cache.sock +production: + url: unix:/path/to/redis.cache.sock diff --git a/spec/fixtures/config/redis_cache_old_format_socket.yml b/spec/fixtures/config/redis_cache_old_format_socket.yml new file mode 100644 index 000000000000..26fa0eda2456 --- /dev/null +++ b/spec/fixtures/config/redis_cache_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.cache.sock +test: unix:/path/to/old/redis.cache.sock +production: unix:/path/to/old/redis.cache.sock diff --git a/spec/fixtures/config/redis_caching_config_with_env.yml b/spec/fixtures/config/redis_caching_config_with_env.yml deleted file mode 100644 index 04ca455a543e..000000000000 --- a/spec/fixtures/config/redis_caching_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_REDIS_CACHING_URL'] %> diff --git a/spec/fixtures/config/redis_caching_new_format_socket.yml b/spec/fixtures/config/redis_caching_new_format_socket.yml deleted file mode 100644 index 9f6d4515f1c7..000000000000 --- a/spec/fixtures/config/redis_caching_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/redis.caching.sock -test: - url: unix:/path/to/redis.caching.sock -production: - url: unix:/path/to/redis.caching.sock diff --git a/spec/fixtures/config/redis_caching_old_format_socket.yml b/spec/fixtures/config/redis_caching_old_format_socket.yml deleted file mode 100644 index c201cc856614..000000000000 --- a/spec/fixtures/config/redis_caching_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/redis.caching.sock -test: unix:/path/to/old/redis.caching.sock -production: unix:/path/to/old/redis.caching.sock diff --git a/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb new file mode 100644 index 000000000000..3693f52b51b7 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::CacheCheck do + include_examples 'simple_check', 'redis_cache_ping', 'RedisCache', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb deleted file mode 100644 index 0e66bd04303f..000000000000 --- a/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative '../simple_check_shared' - -describe Gitlab::HealthChecks::Redis::CachingCheck do - include_examples 'simple_check', 'redis_caching_ping', 'RedisCaching', 'PONG' -end -- GitLab From c1af2ecd762c625e74f3b4facbe1970e05e4ba83 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 14:26:25 +0800 Subject: [PATCH 27/62] rename redis_caching to redis_cache --- spec/controllers/health_controller_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index f9e4171791a2..a1ea1c295ea7 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,7 +19,7 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') expect(json_response['queues_check']['status']).to eq('ok') expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') @@ -44,7 +44,7 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') expect(json_response['queues_check']['status']).to eq('ok') expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') -- GitLab From 27100e95980ee21c78e9cbbde9862029380a4f96 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 15:35:07 +0800 Subject: [PATCH 28/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 * https://gitlab.com/gitlab-org/gitlab-ce/issues/33113 --- .gitignore | 3 +- .gitlab-ci.yml | 108 ++++++++++-------- app/controllers/health_controller.rb | 2 +- app/models/ci/runner.rb | 4 +- app/models/project.rb | 6 +- config/application.rb | 6 +- config/initializers/7_redis.rb | 2 +- ...ng.yml.example => redis.cache.yml.example} | 4 +- config/redis.queues.yml.example | 2 - config/redis.shared_state.yml.example | 2 - config/resque.yml.example | 12 +- lib/gitlab/auth/unique_ips_limiter.rb | 4 +- .../cache/ci/project_pipeline_status.rb | 24 ++-- lib/gitlab/chat_name_token.rb | 12 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/etag_caching/store.rb | 6 +- lib/gitlab/exclusive_lease.rb | 12 +- .../{caching_check.rb => cache_check.rb} | 6 +- lib/gitlab/lfs_token.rb | 6 +- lib/gitlab/mail_room.rb | 1 - lib/gitlab/redis/{caching.rb => cache.rb} | 10 +- lib/gitlab/user_activities.rb | 12 +- lib/gitlab/workhorse.rb | 8 +- lib/tasks/cache.rake | 6 +- locale/gitlab.pot | 4 +- scripts/prepare_build.sh | 6 +- spec/controllers/health_controller_spec.rb | 10 +- .../config/redis_cache_config_with_env.yml | 2 + ...st.yml => redis_cache_new_format_host.yml} | 0 .../config/redis_cache_new_format_socket.yml | 6 + ...st.yml => redis_cache_old_format_host.yml} | 0 .../config/redis_cache_old_format_socket.yml | 3 + .../config/redis_caching_config_with_env.yml | 2 - .../redis_caching_new_format_socket.yml | 6 - .../redis_caching_old_format_socket.yml | 3 - .../cache/ci/project_pipeline_status_spec.rb | 56 ++++----- spec/lib/gitlab/current_settings_spec.rb | 2 +- .../health_checks/redis/cache_check_spec.rb | 6 + .../health_checks/redis/caching_check_spec.rb | 6 - .../redis/{caching_spec.rb => cache_spec.rb} | 32 +++--- spec/lib/gitlab/user_activities_spec.rb | 40 +++---- ...tivities_to_users_last_activity_on_spec.rb | 4 +- spec/models/ci/runner_spec.rb | 8 +- spec/services/git_push_service_spec.rb | 4 +- spec/spec_helper.rb | 6 +- .../unique_ip_check_shared_examples.rb | 2 +- 46 files changed, 233 insertions(+), 235 deletions(-) rename config/{redis.caching.yml.example => redis.cache.yml.example} (85%) rename lib/gitlab/health_checks/redis/{caching_check.rb => cache_check.rb} (78%) rename lib/gitlab/redis/{caching.rb => cache.rb} (59%) create mode 100644 spec/fixtures/config/redis_cache_config_with_env.yml rename spec/fixtures/config/{redis_caching_new_format_host.yml => redis_cache_new_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_cache_new_format_socket.yml rename spec/fixtures/config/{redis_caching_old_format_host.yml => redis_cache_old_format_host.yml} (100%) create mode 100644 spec/fixtures/config/redis_cache_old_format_socket.yml delete mode 100644 spec/fixtures/config/redis_caching_config_with_env.yml delete mode 100644 spec/fixtures/config/redis_caching_new_format_socket.yml delete mode 100644 spec/fixtures/config/redis_caching_old_format_socket.yml create mode 100644 spec/lib/gitlab/health_checks/redis/cache_check_spec.rb delete mode 100644 spec/lib/gitlab/health_checks/redis/caching_check_spec.rb rename spec/lib/gitlab/redis/{caching_spec.rb => cache_spec.rb} (80%) diff --git a/.gitignore b/.gitignore index bf7dadf5cc3f..0490646183e0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ eslint-report.html /config/initializers/smtp_settings.rb /config/initializers/relative_url.rb /config/resque.yml -/config/redis.caching.yml +/config/redis.cache.yml /config/redis.queues.yml /config/redis.shared_state.yml /config/unicorn.rb @@ -61,5 +61,4 @@ eslint-report.html /shared/* /.gitlab_workhorse_secret /webpack-report/ -/locale/gitlab.pot /locale/**/LC_MESSAGES diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc8d0dba05e4..6e861d9d822d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,39 +54,11 @@ stages: services: - postgres:9.2 - redis:alpine -# -# notes for when CI supports service configuration on shared image -# - caching: -# image: redis:alpine -# ports: -# - "6380:6379" -# - queues: -# image: redis:alpine -# ports: -# - "6381:6379" -# - redis_shared_state: -# image: redis:alpine -# ports: -# - "6382:6379" .use-mysql: &use-mysql services: - mysql:latest - redis:alpine -# -# notes for when CI supports service configuration on shared image -# - caching: -# image: redis:alpine -# ports: -# - "6380:6379" -# - queues: -# image: redis:alpine -# ports: -# - "6381:6379" -# - redis_shared_state: -# image: redis:alpine -# ports: -# - "6382:6379" .only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql only: @@ -230,26 +202,66 @@ setup-test-env: - public/assets - tmp/tests -rspec-pg 0 20: *rspec-knapsack-pg -rspec-pg 1 20: *rspec-knapsack-pg -rspec-pg 2 20: *rspec-knapsack-pg -rspec-pg 3 20: *rspec-knapsack-pg -rspec-pg 4 20: *rspec-knapsack-pg -rspec-pg 5 20: *rspec-knapsack-pg -rspec-pg 6 20: *rspec-knapsack-pg -rspec-pg 7 20: *rspec-knapsack-pg -rspec-pg 8 20: *rspec-knapsack-pg -rspec-pg 9 20: *rspec-knapsack-pg -rspec-pg 10 20: *rspec-knapsack-pg -rspec-pg 11 20: *rspec-knapsack-pg -rspec-pg 12 20: *rspec-knapsack-pg -rspec-pg 13 20: *rspec-knapsack-pg -rspec-pg 14 20: *rspec-knapsack-pg -rspec-pg 15 20: *rspec-knapsack-pg -rspec-pg 16 20: *rspec-knapsack-pg -rspec-pg 17 20: *rspec-knapsack-pg -rspec-pg 18 20: *rspec-knapsack-pg -rspec-pg 19 20: *rspec-knapsack-pg +rspec-pg 0 60: *rspec-knapsack-pg +rspec-pg 1 60: *rspec-knapsack-pg +rspec-pg 2 60: *rspec-knapsack-pg +rspec-pg 3 60: *rspec-knapsack-pg +rspec-pg 4 60: *rspec-knapsack-pg +rspec-pg 5 60: *rspec-knapsack-pg +rspec-pg 6 60: *rspec-knapsack-pg +rspec-pg 7 60: *rspec-knapsack-pg +rspec-pg 8 60: *rspec-knapsack-pg +rspec-pg 9 60: *rspec-knapsack-pg +rspec-pg 10 60: *rspec-knapsack-pg +rspec-pg 11 60: *rspec-knapsack-pg +rspec-pg 12 60: *rspec-knapsack-pg +rspec-pg 13 60: *rspec-knapsack-pg +rspec-pg 14 60: *rspec-knapsack-pg +rspec-pg 15 60: *rspec-knapsack-pg +rspec-pg 16 60: *rspec-knapsack-pg +rspec-pg 17 60: *rspec-knapsack-pg +rspec-pg 18 60: *rspec-knapsack-pg +rspec-pg 19 60: *rspec-knapsack-pg +rspec-pg 20 60: *rspec-knapsack-pg +rspec-pg 21 60: *rspec-knapsack-pg +rspec-pg 22 60: *rspec-knapsack-pg +rspec-pg 23 60: *rspec-knapsack-pg +rspec-pg 24 60: *rspec-knapsack-pg +rspec-pg 25 60: *rspec-knapsack-pg +rspec-pg 26 60: *rspec-knapsack-pg +rspec-pg 27 60: *rspec-knapsack-pg +rspec-pg 28 60: *rspec-knapsack-pg +rspec-pg 29 60: *rspec-knapsack-pg +rspec-pg 30 60: *rspec-knapsack-pg +rspec-pg 31 60: *rspec-knapsack-pg +rspec-pg 32 60: *rspec-knapsack-pg +rspec-pg 33 60: *rspec-knapsack-pg +rspec-pg 34 60: *rspec-knapsack-pg +rspec-pg 35 60: *rspec-knapsack-pg +rspec-pg 36 60: *rspec-knapsack-pg +rspec-pg 37 60: *rspec-knapsack-pg +rspec-pg 38 60: *rspec-knapsack-pg +rspec-pg 39 60: *rspec-knapsack-pg +rspec-pg 40 60: *rspec-knapsack-pg +rspec-pg 41 60: *rspec-knapsack-pg +rspec-pg 42 60: *rspec-knapsack-pg +rspec-pg 43 60: *rspec-knapsack-pg +rspec-pg 44 60: *rspec-knapsack-pg +rspec-pg 45 60: *rspec-knapsack-pg +rspec-pg 46 60: *rspec-knapsack-pg +rspec-pg 47 60: *rspec-knapsack-pg +rspec-pg 48 60: *rspec-knapsack-pg +rspec-pg 49 60: *rspec-knapsack-pg +rspec-pg 50 60: *rspec-knapsack-pg +rspec-pg 51 60: *rspec-knapsack-pg +rspec-pg 52 60: *rspec-knapsack-pg +rspec-pg 53 60: *rspec-knapsack-pg +rspec-pg 54 60: *rspec-knapsack-pg +rspec-pg 55 60: *rspec-knapsack-pg +rspec-pg 56 60: *rspec-knapsack-pg +rspec-pg 57 60: *rspec-knapsack-pg +rspec-pg 58 60: *rspec-knapsack-pg +rspec-pg 59 60: *rspec-knapsack-pg rspec-mysql 0 20: *rspec-knapsack-mysql rspec-mysql 1 20: *rspec-knapsack-mysql diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index c1421de14fda..3d061af80540 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,7 +4,7 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::Redis::CachingCheck, + Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d2be57e36ff0..05bca7fea10a 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -145,8 +145,8 @@ def is_runner_queue_value_latest?(value) private def cleanup_runner_queue - Gitlab::Redis::Queues.with do |redis_queues| - redis_queues.del(runner_queue_key) + Gitlab::Redis::Queues.with do |redis| + redis.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index e865409e6f2c..60dee71d4038 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1279,15 +1279,15 @@ def append_or_update_attribute(name, value) end def pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(pushes_since_gc_redis_shared_state_key).to_i } + Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.incr(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.del(pushes_since_gc_redis_shared_state_key) } + Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) diff --git a/config/application.rb b/config/application.rb index af78c06e3219..52799069673b 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,7 +6,7 @@ module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis/caching') + require_dependency Rails.root.join('lib/gitlab/redis/cache') require_dependency Rails.root.join('lib/gitlab/redis/queues') require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') @@ -141,8 +141,8 @@ class Application < Rails::Application end # Use caching across all environments - caching_config_hash = Gitlab::Redis::Caching.params - caching_config_hash[:namespace] = Gitlab::Redis::Caching::CACHE_NAMESPACE + caching_config_hash = Gitlab::Redis::Cache.params + caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 7ad600aa26ae..95ab07e34fad 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,5 @@ # Make sure we initialize a Redis connection pool before Sidekiq starts # multi-threaded execution. -Gitlab::Redis::Caching.with { nil } +Gitlab::Redis::Cache.with { nil } Gitlab::Redis::Queues.with { nil } Gitlab::Redis::SharedState.with { nil } diff --git a/config/redis.caching.yml.example b/config/redis.cache.yml.example similarity index 85% rename from config/redis.caching.yml.example rename to config/redis.cache.yml.example index 8279ddd0c8cd..27478f0a93e4 100644 --- a/config/redis.caching.yml.example +++ b/config/redis.cache.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/10 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6380 # sentinels: # - @@ -16,11 +15,10 @@ development: test: url: redis://localhost:6379/10 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6380 production: # Redis (single instance) - url: unix:/var/run/redis/redis.caching.sock + url: unix:/var/run/redis/redis.cache.sock ## # Redis + Sentinel (for HA) # diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example index 3c954c13fe76..dab1f26b0964 100644 --- a/config/redis.queues.yml.example +++ b/config/redis.queues.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/11 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6381 # sentinels: # - @@ -16,7 +15,6 @@ development: test: url: redis://localhost:6379/11 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6381 production: # Redis (single instance) diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example index 21bc791024fd..9371e3619b78 100644 --- a/config/redis.shared_state.yml.example +++ b/config/redis.shared_state.yml.example @@ -4,7 +4,6 @@ development: url: redis://localhost:6379/12 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6382 # sentinels: # - @@ -16,7 +15,6 @@ development: test: url: redis://localhost:6379/12 # - # notes for when CI supports service configuration on shared image # url: redis://localhost:6382 production: # Redis (single instance) diff --git a/config/resque.yml.example b/config/resque.yml.example index e983be5b32e4..0c19d8bc1d36 100644 --- a/config/resque.yml.example +++ b/config/resque.yml.example @@ -3,24 +3,18 @@ # development: url: redis://localhost:6379 - # - # notes for when CI supports service configuration on shared image - # url: redis://localhost:6379 # sentinels: # - # host: localhost - # port: 26379 # point to sentinel, not to redis port + # port: 26380 # point to sentinel, not to redis port # - # host: slave2 - # port: 26379 # point to sentinel, not to redis port + # port: 26381 # point to sentinel, not to redis port test: url: redis://localhost:6379 - # - # notes for when CI supports service configuration on shared image - # url: redis://localhost:6379 production: # Redis (single instance) - url: unix:/var/run/redis/redis.caching.sock + url: unix:/var/run/redis/redis.sock ## # Redis + Sentinel (for HA) # diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 6fff4309e893..baa1f802d8a0 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,9 +27,9 @@ def update_and_return_ips_count(user_id, ip) time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis::SharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil - redis_shared_state.multi do |r| + redis.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) unique_ips_count = r.zcard(key) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 492e1254f164..05df66c1a43f 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,12 +23,12 @@ def self.load_in_batch_for_projects(projects) end def self.cached_results_for_projects(projects) - result = Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.multi do + result = Gitlab::Redis::Cache.with do |redis| + redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) - redis_caching.exists(cache_key) - redis_caching.hmget(cache_key, :sha, :status, :ref) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) end end end @@ -100,28 +100,28 @@ def store_in_cache_if_needed end def load_from_cache - Gitlab::Redis::Caching.with do |redis_caching| - self.sha, self.status, self.ref = redis_caching.hmget(cache_key, :sha, :status, :ref) + Gitlab::Redis::Cache.with do |redis| + self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::Redis::Cache.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.del(cache_key) + Gitlab::Redis::Cache.with do |redis| + redis.del(cache_key) end end def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.exists(cache_key) + Gitlab::Redis::Cache.with do |redis| + redis.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index bf8d442140fc..e63e54373313 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ def initialize(token = new_token) end def get - Gitlab::Redis::SharedState.with do |redis_shared_state| - data = redis_shared_state.get(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis::SharedState.with do |redis_shared_state| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis_shared_state.set(redis_shared_state_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.del(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 88fd408ef475..f659a975022e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::Redis::Caching::BaseError, ::Errno::ENOENT + rescue ::Gitlab::Redis::Cache::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 7096be42beab..309a542595fb 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -5,14 +5,14 @@ class Store SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis::SharedState.with { |redis_shared_state| redis_shared_state.get(redis_shared_state_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 19f3b7b50b74..c60ca532334a 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -18,8 +18,8 @@ class ExclusiveLease EOS def self.cancel(key, uuid) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end @@ -37,15 +37,15 @@ def initialize(key, timeout:) # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.exists(@redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/health_checks/redis/caching_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb similarity index 78% rename from lib/gitlab/health_checks/redis/caching_check.rb rename to lib/gitlab/health_checks/redis/cache_check.rb index a3c1780e64bc..8068276ac5d1 100644 --- a/lib/gitlab/health_checks/redis/caching_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -1,14 +1,14 @@ module Gitlab module HealthChecks module Redis - class CachingCheck + class CacheCheck extend SimpleAbstractCheck class << self private def metric_prefix - 'redis_caching_ping' + 'redis_cache_ping' end def is_successful?(result) @@ -17,7 +17,7 @@ def is_successful?(result) def check catch_timeout 10.seconds do - Gitlab::Redis::Caching.with(&:ping) + Gitlab::Redis::Cache.with(&:ping) end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 043bfe18545b..8e57ba831c51 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ def initialize(actor) end def token - Gitlab::Redis::SharedState.with do |redis_shared_state| - token = redis_shared_state.get(redis_shared_state_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis_shared_state.set(redis_shared_state_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index a2b363e54085..9f432673a6ec 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -37,7 +37,6 @@ def fetch_config gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) config[:redis_url] = gitlab_redis_queues.url - # @@ TODO this is too-much-information on topology in the client of an object if gitlab_redis_queues.sentinels? config[:sentinels] = gitlab_redis_queues.sentinels end diff --git a/lib/gitlab/redis/caching.rb b/lib/gitlab/redis/cache.rb similarity index 59% rename from lib/gitlab/redis/caching.rb rename to lib/gitlab/redis/cache.rb index b9a0149cf880..62032a0e1ca1 100644 --- a/lib/gitlab/redis/caching.rb +++ b/lib/gitlab/redis/cache.rb @@ -4,9 +4,9 @@ module Gitlab module Redis - class Caching < ::Gitlab::Redis::Wrapper + class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze - DEFAULT_REDIS_CACHING_URL = 'redis://localhost:6380'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze class << self delegate :_raw_config, :fetch_config, :config_file, to: :new @@ -14,9 +14,9 @@ class << self def initialize(rails_env = nil) @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_CACHING_URL - @environment_var_name = 'GITLAB_REDIS_CACHING_CONFIG_FILE'.freeze - @config_file_name = 'redis.caching.yml'.freeze + @default_url = DEFAULT_REDIS_CACHE_URL + @environment_var_name = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + @config_file_name = 'redis.cache.yml'.freeze end end end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index cec80becfb95..125488536e19 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,14 +6,14 @@ class UserActivities BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hset(KEY, key, time.to_i) + Gitlab::Redis::SharedState.with do |redis| + redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hdel(KEY, keys) + Gitlab::Redis::SharedState.with do |redis| + redis.hdel(KEY, keys) end end @@ -21,8 +21,8 @@ def each cursor = 0 loop do cursor, pairs = - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.hscan(KEY, cursor, count: BATCH_SIZE) + Gitlab::Redis::SharedState.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) end Hash[pairs].each { |pair| yield pair } diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 80bd9ca757fd..256836d9716b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,13 +174,13 @@ def secret_path end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis_queues| - result = redis_queues.set(key, value, ex: expire, nx: !overwrite) + Gitlab::Redis::Queues.with do |redis| + result = redis.set(key, value, ex: expire, nx: !overwrite) if result - redis_queues.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") value else - redis_queues.get(key) + redis.get(key) end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 6416b45d5ef7..1b061b5e4ff5 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,17 +1,17 @@ namespace :cache do namespace :clear do - # @@ TODO - push implementation into Gitlab::Redis::Caching + # @@ TODO - push implementation into Gitlab::Redis::Cache KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear key/value cache" task gitlab_kv_cache: :environment do - Gitlab::Redis::Caching.with do |kv_cache| + Gitlab::Redis::Cache.with do |kv_cache| cursor = KV_CACHE_SCAN_START_STOP loop do cursor, keys = kv_cache.scan( cursor, - match: "#{Gitlab::Redis::Caching::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: KV_CACHE_CLEAR_BATCH_SIZE ) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5a2aff5fffc9..3967d40ea9e5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8,8 +8,8 @@ msgid "" msgstr "" "Project-Id-Version: gitlab 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-05-25 12:34+0800\n" -"PO-Revision-Date: 2017-05-25 12:34+0800\n" +"POT-Creation-Date: 2017-05-04 19:24-0500\n" +"PO-Revision-Date: 2017-05-04 19:24-0500\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index a4e3cefbcaa9..9088a387d6a5 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -34,11 +34,11 @@ else # Assume it's mysql sed -i 's/# host:.*/host: mysql/g' config/database.yml fi -cp config/redis.caching.yml.example config/redis.caching.yml -sed -i 's/localhost/redis/g' config/redis.caching.yml +cp config/redis.cache.yml.example config/redis.cache.yml +sed -i 's/localhost/redis/g' config/redis.cache.yml # # notes for when CI supports service configuration on shared image -#sed -i 's/localhost/caching/g' config/redis.caching.yml +#sed -i 's/localhost/caching/g' config/redis.cache.yml cp config/redis.queues.yml.example config/redis.queues.yml sed -i 's/localhost/redis/g' config/redis.queues.yml diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index 13cb333be2c5..a1ea1c295ea7 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,7 +19,7 @@ it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') expect(json_response['queues_check']['status']).to eq('ok') expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') @@ -44,7 +44,7 @@ it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['caching_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') expect(json_response['queues_check']['status']).to eq('ok') expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') @@ -74,9 +74,9 @@ it 'returns Caching ping metrics' do get :metrics - expect(response.body).to match(/^redis_caching_ping_timeout 0$/) - expect(response.body).to match(/^redis_caching_ping_success 1$/) - expect(response.body).to match(/^redis_caching_ping_latency [0-9\.]+$/) + expect(response.body).to match(/^redis_cache_ping_timeout 0$/) + expect(response.body).to match(/^redis_cache_ping_success 1$/) + expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/) end it 'returns Queues ping metrics' do diff --git a/spec/fixtures/config/redis_cache_config_with_env.yml b/spec/fixtures/config/redis_cache_config_with_env.yml new file mode 100644 index 000000000000..52fd5a064605 --- /dev/null +++ b/spec/fixtures/config/redis_cache_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %> diff --git a/spec/fixtures/config/redis_caching_new_format_host.yml b/spec/fixtures/config/redis_cache_new_format_host.yml similarity index 100% rename from spec/fixtures/config/redis_caching_new_format_host.yml rename to spec/fixtures/config/redis_cache_new_format_host.yml diff --git a/spec/fixtures/config/redis_cache_new_format_socket.yml b/spec/fixtures/config/redis_cache_new_format_socket.yml new file mode 100644 index 000000000000..3634c5501633 --- /dev/null +++ b/spec/fixtures/config/redis_cache_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.cache.sock +test: + url: unix:/path/to/redis.cache.sock +production: + url: unix:/path/to/redis.cache.sock diff --git a/spec/fixtures/config/redis_caching_old_format_host.yml b/spec/fixtures/config/redis_cache_old_format_host.yml similarity index 100% rename from spec/fixtures/config/redis_caching_old_format_host.yml rename to spec/fixtures/config/redis_cache_old_format_host.yml diff --git a/spec/fixtures/config/redis_cache_old_format_socket.yml b/spec/fixtures/config/redis_cache_old_format_socket.yml new file mode 100644 index 000000000000..26fa0eda2456 --- /dev/null +++ b/spec/fixtures/config/redis_cache_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.cache.sock +test: unix:/path/to/old/redis.cache.sock +production: unix:/path/to/old/redis.cache.sock diff --git a/spec/fixtures/config/redis_caching_config_with_env.yml b/spec/fixtures/config/redis_caching_config_with_env.yml deleted file mode 100644 index 04ca455a543e..000000000000 --- a/spec/fixtures/config/redis_caching_config_with_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -test: - url: <%= ENV['TEST_GITLAB_REDIS_CACHING_URL'] %> diff --git a/spec/fixtures/config/redis_caching_new_format_socket.yml b/spec/fixtures/config/redis_caching_new_format_socket.yml deleted file mode 100644 index 9f6d4515f1c7..000000000000 --- a/spec/fixtures/config/redis_caching_new_format_socket.yml +++ /dev/null @@ -1,6 +0,0 @@ -development: - url: unix:/path/to/redis.caching.sock -test: - url: unix:/path/to/redis.caching.sock -production: - url: unix:/path/to/redis.caching.sock diff --git a/spec/fixtures/config/redis_caching_old_format_socket.yml b/spec/fixtures/config/redis_caching_old_format_socket.yml deleted file mode 100644 index c201cc856614..000000000000 --- a/spec/fixtures/config/redis_caching_old_format_socket.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: unix:/path/to/old/redis.caching.sock -test: unix:/path/to/old/redis.caching.sock -production: unix:/path/to/old/redis.caching.sock diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index e6bd6b276fe7..a6741f16113e 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_caching do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do let!(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } let(:cache_key) { "projects/#{project.id}/pipeline_status" } @@ -28,8 +28,8 @@ expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) end - describe 'without a status in redis_caching' do - it 'loads the status from a commit when it was not in redis_caching' do + describe 'without a status in redis_cache' do + it 'loads the status from a commit when it was not in redis_cache' do empty_status = { sha: nil, status: nil, ref: nil } fake_pipeline = described_class.new( project_without_status, @@ -48,9 +48,9 @@ described_class.load_in_batch_for_projects([project_without_status]) end - it 'only connects to redis_caching twice' do + it 'only connects to redis_cache twice' do # Once to load, once to store in the cache - expect(Gitlab::Redis::Caching).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -58,10 +58,10 @@ end end - describe 'when a status was cached in redis_caching' do + describe 'when a status was cached in redis_cache' do before do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Cache.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end @@ -76,8 +76,8 @@ expect(pipeline_status.ref).to eq(ref) end - it 'only connects to redis_caching once' do - expect(Gitlab::Redis::Caching).to receive(:with).exactly(1).and_call_original + it 'only connects to redis_cache once' do + expect(Gitlab::Redis::Cache).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -95,8 +95,8 @@ describe '.cached_results_for_projects' do it 'loads a status from caching for all projects' do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + Gitlab::Redis::Cache.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end result = [{ loaded_from_cache: false, pipeline_info: { sha: nil, status: nil, ref: nil } }, @@ -183,7 +183,7 @@ end end - describe "#load_from_project", :clean_gitlab_redis_caching do + describe "#load_from_project", :clean_gitlab_redis_cache do let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } it 'reads the status from the pipeline for the commit' do @@ -203,40 +203,40 @@ end end - describe "#store_in_cache", :clean_gitlab_redis_caching do + describe "#store_in_cache", :clean_gitlab_redis_cache do it "sets the object in caching" do pipeline_status.sha = '123456' pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') end end - describe '#store_in_cache_if_needed', :clean_gitlab_redis_caching do + describe '#store_in_cache_if_needed', :clean_gitlab_redis_cache do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil expect(ref).not_to be_nil end - it "doesn't store the status in redis_caching when the sha is not the head of the project" do + it "doesn't store the status in redis_cache when the sha is not the head of the project" do other_status = described_class.new( project, pipeline_info: { sha: "123456", status: "failed" } ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,8 +244,8 @@ it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Cache.with do |redis| + redis.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -263,20 +263,20 @@ end end - describe "with a status in caching", :clean_gitlab_redis_caching do + describe "with a status in caching", :clean_gitlab_redis_cache do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:ref) { 'master' } before do - Gitlab::Redis::Caching.with do |redis_caching| - redis_caching.mapped_hmset(cache_key, + Gitlab::Redis::Cache.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end describe '#load_from_cache' do - it 'reads the status from redis_caching' do + it 'reads the status from redis_cache' do pipeline_status.load_from_cache expect(pipeline_status.sha).to eq(sha) @@ -292,10 +292,10 @@ end describe '#delete_from_cache' do - it 'deletes values from redis_caching' do + it 'deletes values from redis_cache' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis::Caching.with { |redis_caching| redis_caching.exists(cache_key) } + key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index a77a24de5aef..55fb547b8dd2 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Caching::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Cache::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb new file mode 100644 index 000000000000..3693f52b51b7 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::CacheCheck do + include_examples 'simple_check', 'redis_cache_ping', 'RedisCache', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb b/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb deleted file mode 100644 index 0e66bd04303f..000000000000 --- a/spec/lib/gitlab/health_checks/redis/caching_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative '../simple_check_shared' - -describe Gitlab::HealthChecks::Redis::CachingCheck do - include_examples 'simple_check', 'redis_caching_ping', 'RedisCaching', 'PONG' -end diff --git a/spec/lib/gitlab/redis/caching_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb similarity index 80% rename from spec/lib/gitlab/redis/caching_spec.rb rename to spec/lib/gitlab/redis/cache_spec.rb index 1ff8dc1e78a8..3f36e2164b5e 100644 --- a/spec/lib/gitlab/redis/caching_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe Gitlab::Redis::Caching do +describe Gitlab::Redis::Cache do include StubENV - let(:config) { 'config/redis.caching.yml' } + let(:config) { 'config/redis.cache.yml' } before(:each) do - stub_env('GITLAB_REDIS_CACHING_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env('GITLAB_REDIS_CACHE_CONFIG_FILE', Rails.root.join(config).to_s) clear_raw_config end @@ -26,14 +26,14 @@ end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_socket.yml' } + let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_socket.yml' } + let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_socket.yml' } context 'with old format' do let(:config) { config_old } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.caching.sock') + is_expected.to include(path: '/path/to/old/redis.cache.sock') is_expected.not_to have_key(:url) end end @@ -42,15 +42,15 @@ let(:config) { config_new } it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.caching.sock') + is_expected.to include(path: '/path/to/redis.cache.sock') is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } + let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } context 'with old format' do let(:config) { config_old } @@ -82,10 +82,10 @@ end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_caching_config_with_env.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_config_with_env.yml' } before do - stub_env('TEST_GITLAB_REDIS_CACHING_URL', 'redis://redishost:6380') + stub_env('TEST_GITLAB_REDIS_CACHE_URL', 'redis://redishost:6380') end it 'reads caching url from env variable' do @@ -139,7 +139,7 @@ subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } it 'returns an array of hashes with host and port keys' do is_expected.to include(host: 'localhost', port: 26380) @@ -148,7 +148,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } it 'returns nil' do is_expected.to be_nil @@ -160,7 +160,7 @@ subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_caching_new_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } it 'returns true' do is_expected.to be_truthy @@ -168,7 +168,7 @@ end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_caching_old_format_host.yml' } + let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } it 'returns false' do is_expected.to be_falsey @@ -180,7 +180,7 @@ it 'returns default caching url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Caching::DEFAULT_REDIS_CACHING_URL) + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL) end it 'returns old-style single url config in a hash' do diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index 9a9011044590..a4ea0ac59e96 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -11,8 +11,8 @@ described_class.record(42) end - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -21,8 +21,8 @@ it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end @@ -34,28 +34,28 @@ it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and key does not exist' do it 'removes the pair from SharedState' do - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -67,14 +67,14 @@ described_class.record(41, now) described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end @@ -83,14 +83,14 @@ it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis::SharedState.with do |redis_shared_state| - expect(redis_shared_state.hscan(described_class::KEY, 0)).to eq(['0', []]) + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index d30d7fd92ddb..a7be02f24278 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -9,8 +9,8 @@ let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::Redis::SharedState.with do |redis_shared_state| - redis_shared_state.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) + Gitlab::Redis::SharedState.with do |redis| + redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 678c02c851fa..4b9cce28e0e6 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -311,9 +311,9 @@ end def expect_value_in_queues - Gitlab::Redis::Queues.with do |redis_queues| + Gitlab::Redis::Queues.with do |redis| runner_queue_key = runner.send(:runner_queue_key) - expect(redis_queues.get(runner_queue_key)) + expect(redis.get(runner_queue_key)) end end end @@ -330,8 +330,8 @@ def expect_value_in_queues end it 'cleans up the queue' do - Gitlab::Redis::Queues.with do |redis_queues| - expect(redis_queues.get(queue_key)).to be_nil + Gitlab::Redis::Queues.with do |redis| + expect(redis.get(queue_key)).to be_nil end end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 12468c4a9097..6091fa6f0bab 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -527,7 +527,7 @@ before do # Flush any raw key-value data stored by the housekeeping code. - Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Cache.with { |conn| conn.flushall } Gitlab::Redis::Queues.with { |conn| conn.flushall } Gitlab::Redis::SharedState.with { |conn| conn.flushall } @@ -535,7 +535,7 @@ end after do - Gitlab::Redis::Caching.with { |conn| conn.flushall } + Gitlab::Redis::Cache.with { |conn| conn.flushall } Gitlab::Redis::Queues.with { |conn| conn.flushall } Gitlab::Redis::SharedState.with { |conn| conn.flushall } end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 741aacc0dad7..61a4fbfaa8d0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -85,12 +85,12 @@ Rails.cache = caching_store end - config.around(:each, :clean_gitlab_redis_caching) do |example| - Gitlab::Redis::Caching.with(&:flushall) + config.around(:each, :clean_gitlab_redis_cache) do |example| + Gitlab::Redis::Cache.with(&:flushall) example.run - Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Cache.with(&:flushall) end config.around(:each, :clean_gitlab_redis_shared_state) do |example| diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 6e25c5361916..ee2b7de925b2 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,7 +1,7 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::Redis::Caching.with(&:flushall) + Gitlab::Redis::Cache.with(&:flushall) Gitlab::Redis::Queues.with(&:flushall) Gitlab::Redis::SharedState.with(&:flushall) end -- GitLab From 322fe7b362b1c0e3ffb7ac0543007a2bf0a9cf79 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 18:35:28 +0800 Subject: [PATCH 29/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple Redis key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [ ] Build Support * [ ] Deployment (Omnibus) * [x] Documentation * [ ] using atomic transactions * [ ] using actual Model objects for encapsulation of underlying technology * [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception * [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes * [x] retention policy for different namespaces * [x] abstract class between new KV objects and redis for shared behaviors * [ ] Migration from single instance to multiple instances * [ ] Changelog partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 * https://gitlab.com/gitlab-org/gitlab-ce/issues/33113 --- config/README.md | 128 +++++++++++++++++- .../high_availability/redis_source.md | 5 + 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/config/README.md b/config/README.md index 0a5ea2424e0a..18b2ed5848fa 100644 --- a/config/README.md +++ b/config/README.md @@ -1,6 +1,6 @@ # Configuration files Documentation -Note that most configuration files (`config/*.*`) committed into +Note that most configuration files (`*.*`) committed into [gitlab-ce](https://gitlab.com/gitlab-org/gitlab-ce) **will not be used** for [omnibus-gitlab](https://gitlab.com/gitlab-org/omnibus-gitlab). Configuration files committed into gitlab-ce are only used for development. @@ -19,4 +19,128 @@ an ERB file and then loads the resulting YML as its configuration. This file is called `resque.yml` for historical reasons. We are **NOT** using Resque at the moment. It is used to specify Redis configuration -values instead. +values instead when a single database instance of Redis is desired. + +# Advanced Redis configuration files + +In more advanced configurations of Redis key-value storage, it is desirable +to separate the keys by lifecycle and intended use to ease provisioning and +management of scalable Redis clusters. + +These settings provide routing and other configuration data (such as sentinel, +persistence policies, and other Redis customization) for connections +to Redis single instances, Redis sentinel, and Redis clusters. + +If desired, the routing URL provided by these settings can be used with: +1. Unix Socket + 1. named socket for each Redis instance desired. + 2. `database number` desired for each Redis instance desired. +2. TCP Socket + 1. `host name` or IP for each Redis instance desired + 2. TCP port number for each Redis instance desired + 3. `database number` for each Redis instance desired + +--- +## redis.cache.yml + +If configured, `redis.cache.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for `Rails.cache` and other volatile non-persistent data which enhances +the performance of gitlab. +Settings here can be overridden by the environment variables +`DEFAULT_REDIS_CACHE_URL` which overrides the URL used by the Redis +connection manager, and `GITLAB_REDIS_CACHE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `cache` is: +1. URL from environment variable `DEFAULT_REDIS_CACHE_URL` +2. URL from a configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +3. URL from `redis.cache.yml` +4. URL from environment variable `DEFAULT_REDIS_URL` +5. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +6. URL from `resque.yml` + +The order of precedence for all other configuration settings for `cache` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +2. the configuration file `redis.cache.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +--- +## redis.queues.yml +If configured, `redis.queues.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::Queues`. +These queues are intended to be the foundation +of reliable inter-process communication between modules, whether on the same +host node, or within a cluster. The primary clients of the queues are +SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can +be overridden by the environment variables `DEFAULT_REDIS_QUEUES_URL` which + overrides the URL used by the Redis connection manager, and +`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for +configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `queues` is: +1. URL from environment variable `DEFAULT_REDIS_QUEUES_URL` +2. URL from a configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +3. URL from `redis.queues.yml` +4. URL from environment variable `DEFAULT_REDIS_URL` +5. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +6. URL from `resque.yml` + +The order of precedence for all other configuration settings for `queues` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +2. the configuration file `redis.queues.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +--- +## redis.shared_state.yml + +If configured, `redis.shared_state.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::SharedState` such as session state, +and rate limiting. +Settings here can be overridden by the environment variables +`DEFAULT_REDIS_SHARED_STATE_URL` which overrides the URL used by the Redis +connection manager, and `GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `shared_state` is: +1. URL from environment variable `DEFAULT_REDIS_SHARED_STATE_URL` +2. URL from a configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +3. URL from `redis.shared_state.yml` +4. URL from environment variable `DEFAULT_REDIS_URL` +5. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +6. URL from `resque.yml` + +The order of precedence for all other configuration settings for `shared_state` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +2. the configuration file `redis.shared_state.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index 3629772b8af3..ee7df5f33cca 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when you have installed Redis all by yourself and not using the bundled one that comes with the Omnibus packages. +Note also that you may elect to override all references to +`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis +settings outlined in +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md). + We cannot stress enough the importance of reading the [Overview section](redis.md#overview) of the Omnibus Redis HA as it provides some invaluable information to the configuration of Redis. Please proceed to -- GitLab From 6cbda07ac45146256de06387e7a33fca6e8665f1 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 20:37:16 +0800 Subject: [PATCH 30/62] use ::Redis exceptions --- config/initializers/sidekiq.rb | 2 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/redis/wrapper.rb | 2 -- spec/lib/gitlab/current_settings_spec.rb | 2 +- spec/services/projects/destroy_service_spec.rb | 4 ++-- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index bf99b248bbcb..85c5cc4c9ece 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -74,5 +74,5 @@ end end end -rescue Gitlab::Redis::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index f659a975022e..82576d197fed 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::Redis::Cache::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 011708b8b26f..4551c06bc7cf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,8 +8,6 @@ module Gitlab module Redis class Wrapper - BaseError = ::Redis::BaseError.freeze - CannotConnectError = ::Redis::CannotConnectError.freeze DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze CONFIG_FILE_NAME = 'resque.yml'.freeze diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index 55fb547b8dd2..404fed34ced0 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Cache::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Redis::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 114357ff5c77..a02d3b1be648 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::Redis::SharedState::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(::Redis::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Gitlab::Redis::SharedState::CannotConnectError + rescue ::Redis::CannotConnectError end end -- GitLab From 7b9c12da128fe51120b51d80f1fe682d96c665db Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 21:31:50 +0800 Subject: [PATCH 31/62] better documentation for initializers --- config/initializers/7_redis.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 95ab07e34fad..af4967521b80 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,8 @@ -# Make sure we initialize a Redis connection pool before Sidekiq starts -# multi-threaded execution. +# Make sure we initialize a Redis connection pool before multi-threaded +# execution starts by +# 1. Sidekiq +# 2. Rails.cache +# 3. HTTP clients Gitlab::Redis::Cache.with { nil } Gitlab::Redis::Queues.with { nil } Gitlab::Redis::SharedState.with { nil } -- GitLab From 2ad959766f71cff9dbae44ec679aa8162433681e Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 31 May 2017 21:41:20 +0800 Subject: [PATCH 32/62] WIP: RFC https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple Redis key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured areas that need some more attention * [x] CI support * [x] Build Support - deferred to issue https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2389 * [x] Deployment (Omnibus) - deferred to issue https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2389 * [x] Documentation * [x] retention policy for different namespaces * [x] abstract class between new KV objects and Redis for shared behaviors * [x] Migration from single instance to multiple instances - deferred to issue XXX ~~* [ ] using atomic transactions~~ - deferred to issue XXX ~~* [ ] using actual Model objects for encapsulation of underlying technology~~ - deferred to issue XXX ~~* [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception~~ -deferred to issue XXX ~~* [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes~~ - deferred to investigation issue XXX ~~* [ ] Changelog~~ - deferred to release management process partially solves: * https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 see also: * [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) required for testing on laptop * https://gitlab.com/gitlab-org/gitlab-ce * https://gitlab.com/gitlab-org/gitlab-ee * https://gitlab.com/gitlab-org/omnibus-gitlab * https://gitlab.com/gitlab-com/infrastructure/issues/1682 * https://gitlab.com/gitlab-com/infrastructure/issues/1631 * https://gitlab.com/gitlab-com/infrastructure/issues/1802 * https://gitlab.com/gitlab-com/infrastructure/issues/1575 * https://gitlab.com/gitlab-org/gitlab-ce/issues/33113 --- config/initializers/7_redis.rb | 7 +++++-- config/initializers/sidekiq.rb | 2 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/redis/wrapper.rb | 2 -- spec/lib/gitlab/current_settings_spec.rb | 2 +- spec/services/projects/destroy_service_spec.rb | 4 ++-- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 95ab07e34fad..af4967521b80 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,5 +1,8 @@ -# Make sure we initialize a Redis connection pool before Sidekiq starts -# multi-threaded execution. +# Make sure we initialize a Redis connection pool before multi-threaded +# execution starts by +# 1. Sidekiq +# 2. Rails.cache +# 3. HTTP clients Gitlab::Redis::Cache.with { nil } Gitlab::Redis::Queues.with { nil } Gitlab::Redis::SharedState.with { nil } diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index bf99b248bbcb..85c5cc4c9ece 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -74,5 +74,5 @@ end end end -rescue Gitlab::Redis::Queues::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index f659a975022e..82576d197fed 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -15,7 +15,7 @@ def ensure_application_settings! begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Gitlab::Redis::Cache::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT settings = ::ApplicationSetting.last end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 011708b8b26f..4551c06bc7cf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,8 +8,6 @@ module Gitlab module Redis class Wrapper - BaseError = ::Redis::BaseError.freeze - CannotConnectError = ::Redis::CannotConnectError.freeze DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze CONFIG_FILE_NAME = 'resque.yml'.freeze diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index 55fb547b8dd2..404fed34ced0 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -27,7 +27,7 @@ end it 'falls back to DB if Caching fails' do - expect(ApplicationSetting).to receive(:current).and_raise(::Gitlab::Redis::Cache::BaseError) + expect(ApplicationSetting).to receive(:current).and_raise(::Redis::BaseError) expect(ApplicationSetting).to receive(:last).and_call_original expect(current_application_settings).to be_a(ApplicationSetting) diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 114357ff5c77..a02d3b1be648 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -59,14 +59,14 @@ before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Gitlab::Redis::SharedState::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(::Redis::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Gitlab::Redis::SharedState::CannotConnectError + rescue ::Redis::CannotConnectError end end -- GitLab From 4352b75668e564e0a650f51b0b3644014c5f1bb9 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 08:02:42 +0800 Subject: [PATCH 33/62] formatting, spelling --- config/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/config/README.md b/config/README.md index 7da2d452bee9..82dc9ae98b41 100644 --- a/config/README.md +++ b/config/README.md @@ -34,19 +34,18 @@ to Redis single instances, Redis sentinel, and Redis clusters. If desired, the routing URL provided by these settings can be used with: 1. Unix Socket 1. named socket for each Redis instance desired. - 2. `database number` desired for each Redis instance desired. + 2. `database number` for each Redis instance desired. 2. TCP Socket 1. `host name` or IP for each Redis instance desired 2. TCP port number for each Redis instance desired 3. `database number` for each Redis instance desired ---- ## redis.cache.yml If configured, `redis.cache.yml` overrides the `resque.yml` settings to configure the Redis database instance used for `Rails.cache` and other volatile non-persistent data which enhances -the performance of gitlab. +the performance of GitLab. Settings here can be overridden by the environment variables `DEFAULT_REDIS_CACHE_URL` which overrides the URL used by the Redis connection manager, and `GITLAB_REDIS_CACHE_CONFIG_FILE` which provides @@ -74,8 +73,8 @@ searched): `GITLAB_REDIS_CONFIG_FILE` environment variable 4. the configuration file `resque.yml` ---- ## redis.queues.yml + If configured, `redis.queues.yml` overrides the `resque.yml` settings to configure the Redis database instance used for clients of `::Gitlab::Redis::Queues`. @@ -110,7 +109,6 @@ searched): `GITLAB_REDIS_CONFIG_FILE` environment variable 4. the configuration file `resque.yml` ---- ## redis.shared_state.yml If configured, `redis.shared_state.yml` overrides the -- GitLab From 7ad14ff11e21d4c3383636d58fe82ca2bd5d4261 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 10:36:31 +0800 Subject: [PATCH 34/62] rework redis health checks to preserve prior behavior due to user-facing contracts for monitoring --- app/controllers/health_controller.rb | 1 + lib/gitlab/health_checks/redis/cache_check.rb | 3 +++ .../health_checks/redis/queues_check.rb | 4 +++ lib/gitlab/health_checks/redis/redis_check.rb | 25 +++++++++++++++++++ .../health_checks/redis/shared_state_check.rb | 4 +++ .../health_checks/redis/redis_check_spec.rb | 6 +++++ 6 files changed, 43 insertions(+) create mode 100644 lib/gitlab/health_checks/redis/redis_check.rb create mode 100644 spec/lib/gitlab/health_checks/redis/redis_check_spec.rb diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 3d061af80540..7d80d16c7296 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,6 +4,7 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, + Gitlab::HealthChecks::Redis::RedisCheck, Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 8068276ac5d1..01daa0f390a8 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -5,6 +5,9 @@ class CacheCheck extend SimpleAbstractCheck class << self + def check_up + check + end private def metric_prefix diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index c61033ae5721..f97d50d39479 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -5,6 +5,10 @@ class QueuesCheck extend SimpleAbstractCheck class << self + def check_up + check + end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 000000000000..e3557dc9208b --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,25 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + CacheCheck.check_up && QueuesCheck.check_up && SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index edd55400c183..e3244392902a 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -5,6 +5,10 @@ class SharedStateCheck extend SimpleAbstractCheck class << self + def check_up + check + end + private def metric_prefix diff --git a/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb new file mode 100644 index 000000000000..03afc1cd761c --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::RedisCheck do + include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' +end -- GitLab From 0ec0e783501d326ca02d9d51c30c6abc15e38f25 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 10:37:24 +0800 Subject: [PATCH 35/62] rework based on feedback from code review --- lib/gitlab/redis/cache.rb | 26 +++++++++---- lib/gitlab/redis/queues.rb | 26 +++++++++---- lib/gitlab/redis/shared_state.rb | 26 +++++++++---- lib/gitlab/redis/wrapper.rb | 66 ++++++++++++++------------------ 4 files changed, 86 insertions(+), 58 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 62032a0e1ca1..54e7b64444fc 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,22 +1,34 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = '../../../config/redis.cache.yml'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_CACHE_URL - @environment_var_name = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - @config_file_name = 'redis.cache.yml'.freeze + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index b45a6a757819..4200f6ef9d64 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,6 +1,6 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis @@ -8,16 +8,28 @@ class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = '../../../config/redis.queues.yml'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_QUEUES_URL - @environment_var_name = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - @config_file_name = 'redis.queues.yml'.freeze + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 89fae79e248a..be7f191b213f 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,22 +1,34 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = '../../../config/redis.shared_state.yml'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_SHARED_STATE_URL - @environment_var_name = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - @config_file_name = 'redis.shared_state.yml'.freeze + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 4551c06bc7cf..b68c6347f371 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -9,11 +9,11 @@ module Gitlab module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - CONFIG_FILE_NAME = 'resque.yml'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + DEFAULT_REDIS_CONFIG_FILE_NAME = '../../../config/resque.yml'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, :params, :url, to: :new + delegate :params, :url, to: :new def with @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @@ -29,25 +29,35 @@ def pool_size end size end - end - def _raw_config - return @_raw_config if defined?(@_raw_config) + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(config_file_name)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false + @_raw_config end - @_raw_config + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + file_name = nil unless File.file?(file_name) + file_name + end end def initialize(rails_env = nil) @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_URL - @environment_var_name = ENV_VAR_NAME - @config_file_name = CONFIG_FILE_NAME end def params @@ -85,41 +95,23 @@ def redis_store_options end end + def default_url + DEFAULT_REDIS_URL + end + def raw_config_hash config_data = fetch_config if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: "#{@default_url}" } + { url: default_url } end end def fetch_config self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end - - def config_file - # if ENV set for sub-class, use it even if it points to a file does not exist - proposed_file_name = ENV[@environment_var_name] - file_name = proposed_file_name if proposed_file_name - # otherwise, if config files exists for sub-class, use it - unless file_name - proposed_file_name = File.expand_path("../../../config/#{@config_file_name}", __dir__) - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - unless file_name - proposed_file_name = ENV[ENV_VAR_NAME] - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - # otherwise, if config files exists for wrapper class, use it - unless file_name - proposed_file_name = File.expand_path("../../../config/#{CONFIG_FILE_NAME}", __dir__) - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - file_name - end end end end -- GitLab From f8692821dab24a8788f048a11f410ff951ef3924 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 12:49:39 +0800 Subject: [PATCH 36/62] address CI reported issues --- lib/gitlab/health_checks/redis/cache_check.rb | 1 + lib/gitlab/health_checks/redis/redis_check.rb | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 01daa0f390a8..a28658d42d43 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -8,6 +8,7 @@ class << self def check_up check end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index e3557dc9208b..9cafef1fe043 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -5,6 +5,7 @@ class RedisCheck extend SimpleAbstractCheck class << self + private def metric_prefix @@ -16,7 +17,9 @@ def is_successful?(result) end def check - CacheCheck.check_up && QueuesCheck.check_up && SharedStateCheck.check_up + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up end end end -- GitLab From ca43b3b8dd0772f7a9e4313533cd14c928a110bf Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 16:08:26 +0800 Subject: [PATCH 37/62] address CI reported issues --- lib/gitlab/health_checks/redis/redis_check.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index 9cafef1fe043..7bdb9e5ec5ac 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -5,7 +5,6 @@ class RedisCheck extend SimpleAbstractCheck class << self - private def metric_prefix @@ -18,8 +17,8 @@ def is_successful?(result) def check ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && \ - ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && \ - ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up end end end -- GitLab From 5a4ce70600c88552aa44b9fc2a3b706c51b494c1 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 16:11:45 +0800 Subject: [PATCH 38/62] https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple Redis key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured --- * this MR is primarily submitted for review of * design approach * scope of changes * size of MR (1000+ lines -- could be broken into a few smaller step-wise pieces) * GDK testing can be performed with [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) * the Gitlab::Redis class was deliberately removed/deprecated to force the runtime to find any uses that were not readily apparent from grep/search of git repository * the modified GDK launches 3 separate KV servers (redis instances at present moment) and wraps each instance in a separate class (Caches, Queues, SharedState), each with bindings to appropriate namespaces within the KV store. * the modified CI launches 1 redis instance, and assigns each KV store to a different `database number` --- areas that need some more attention * [x] CI support * [x] Build Support - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Deployment (Omnibus) - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Developer Support (GDK) - deferred to issue gitlab-org/gitlab-development-kit#238 * [x] Documentation * [x] retention policy for different namespaces * [x] abstract class between new KV objects and Redis for shared behaviors * [x] Migration from single instance to multiple instances - deferred to issue gitlab-org/gitlab-ce#33185 - deferred to issue gitlab-org/gitlab-ce#33181 - ~~* [ ] using atomic transactions~~ - deferred to investigation issue gitlab-org/gitlab-ce#33182 - ~~* [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes~~ - deferred to release management process - ~~* [ ] Changelog~~ - deferred - ~~* [ ] using actual Model objects for encapsulation of underlying technology~~ - deferred - ~~* [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception~~ --- app/controllers/health_controller.rb | 1 + config/README.md | 8 +-- lib/gitlab/health_checks/redis/cache_check.rb | 4 ++ .../health_checks/redis/queues_check.rb | 4 ++ lib/gitlab/health_checks/redis/redis_check.rb | 27 ++++++++ .../health_checks/redis/shared_state_check.rb | 4 ++ lib/gitlab/redis/cache.rb | 26 ++++++-- lib/gitlab/redis/queues.rb | 26 ++++++-- lib/gitlab/redis/shared_state.rb | 26 ++++++-- lib/gitlab/redis/wrapper.rb | 66 ++++++++----------- .../health_checks/redis/redis_check_spec.rb | 6 ++ 11 files changed, 135 insertions(+), 63 deletions(-) create mode 100644 lib/gitlab/health_checks/redis/redis_check.rb create mode 100644 spec/lib/gitlab/health_checks/redis/redis_check_spec.rb diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index 3d061af80540..7d80d16c7296 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,6 +4,7 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, + Gitlab::HealthChecks::Redis::RedisCheck, Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, diff --git a/config/README.md b/config/README.md index 7da2d452bee9..82dc9ae98b41 100644 --- a/config/README.md +++ b/config/README.md @@ -34,19 +34,18 @@ to Redis single instances, Redis sentinel, and Redis clusters. If desired, the routing URL provided by these settings can be used with: 1. Unix Socket 1. named socket for each Redis instance desired. - 2. `database number` desired for each Redis instance desired. + 2. `database number` for each Redis instance desired. 2. TCP Socket 1. `host name` or IP for each Redis instance desired 2. TCP port number for each Redis instance desired 3. `database number` for each Redis instance desired ---- ## redis.cache.yml If configured, `redis.cache.yml` overrides the `resque.yml` settings to configure the Redis database instance used for `Rails.cache` and other volatile non-persistent data which enhances -the performance of gitlab. +the performance of GitLab. Settings here can be overridden by the environment variables `DEFAULT_REDIS_CACHE_URL` which overrides the URL used by the Redis connection manager, and `GITLAB_REDIS_CACHE_CONFIG_FILE` which provides @@ -74,8 +73,8 @@ searched): `GITLAB_REDIS_CONFIG_FILE` environment variable 4. the configuration file `resque.yml` ---- ## redis.queues.yml + If configured, `redis.queues.yml` overrides the `resque.yml` settings to configure the Redis database instance used for clients of `::Gitlab::Redis::Queues`. @@ -110,7 +109,6 @@ searched): `GITLAB_REDIS_CONFIG_FILE` environment variable 4. the configuration file `resque.yml` ---- ## redis.shared_state.yml If configured, `redis.shared_state.yml` overrides the diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 8068276ac5d1..a28658d42d43 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -5,6 +5,10 @@ class CacheCheck extend SimpleAbstractCheck class << self + def check_up + check + end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index c61033ae5721..f97d50d39479 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -5,6 +5,10 @@ class QueuesCheck extend SimpleAbstractCheck class << self + def check_up + check + end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 000000000000..7bdb9e5ec5ac --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index edd55400c183..e3244392902a 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -5,6 +5,10 @@ class SharedStateCheck extend SimpleAbstractCheck class << self + def check_up + check + end + private def metric_prefix diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 62032a0e1ca1..54e7b64444fc 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,22 +1,34 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = '../../../config/redis.cache.yml'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_CACHE_URL - @environment_var_name = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - @config_file_name = 'redis.cache.yml'.freeze + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index b45a6a757819..4200f6ef9d64 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,6 +1,6 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis @@ -8,16 +8,28 @@ class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = '../../../config/redis.queues.yml'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_QUEUES_URL - @environment_var_name = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - @config_file_name = 'redis.queues.yml'.freeze + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 89fae79e248a..be7f191b213f 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,22 +1,34 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: -require_relative 'wrapper' +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = '../../../config/redis.shared_state.yml'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, to: :new + delegate :params, :config_file_name, :url, to: :new end - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_SHARED_STATE_URL - @environment_var_name = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - @config_file_name = 'redis.shared_state.yml'.freeze + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 4551c06bc7cf..b68c6347f371 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -9,11 +9,11 @@ module Gitlab module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - CONFIG_FILE_NAME = 'resque.yml'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + DEFAULT_REDIS_CONFIG_FILE_NAME = '../../../config/resque.yml'.freeze class << self - delegate :_raw_config, :fetch_config, :config_file, :params, :url, to: :new + delegate :params, :url, to: :new def with @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @@ -29,25 +29,35 @@ def pool_size end size end - end - def _raw_config - return @_raw_config if defined?(@_raw_config) + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(config_file_name)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false + @_raw_config end - @_raw_config + def config_file_name + # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + + # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + file_name = nil unless File.file?(file_name) + file_name + end end def initialize(rails_env = nil) @rails_env = rails_env || ::Rails.env - @default_url = DEFAULT_REDIS_URL - @environment_var_name = ENV_VAR_NAME - @config_file_name = CONFIG_FILE_NAME end def params @@ -85,41 +95,23 @@ def redis_store_options end end + def default_url + DEFAULT_REDIS_URL + end + def raw_config_hash config_data = fetch_config if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: "#{@default_url}" } + { url: default_url } end end def fetch_config self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end - - def config_file - # if ENV set for sub-class, use it even if it points to a file does not exist - proposed_file_name = ENV[@environment_var_name] - file_name = proposed_file_name if proposed_file_name - # otherwise, if config files exists for sub-class, use it - unless file_name - proposed_file_name = File.expand_path("../../../config/#{@config_file_name}", __dir__) - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - unless file_name - proposed_file_name = ENV[ENV_VAR_NAME] - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - # otherwise, if config files exists for wrapper class, use it - unless file_name - proposed_file_name = File.expand_path("../../../config/#{CONFIG_FILE_NAME}", __dir__) - file_name = proposed_file_name if proposed_file_name && File.file?(proposed_file_name) - end - file_name - end end end end diff --git a/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb new file mode 100644 index 000000000000..03afc1cd761c --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::RedisCheck do + include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' +end -- GitLab From 9106324ef97092ebce2e75af06da213df4f19ebd Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 18:00:58 +0800 Subject: [PATCH 39/62] https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple Redis key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured --- * this MR is primarily submitted for review of * design approach * scope of changes * size of MR (1000+ lines -- could be broken into a few smaller step-wise pieces) * GDK testing can be performed with [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) * the Gitlab::Redis class was deliberately removed/deprecated to force the runtime to find any uses that were not readily apparent from grep/search of git repository * the modified GDK launches 3 separate KV servers (redis instances at present moment) and wraps each instance in a separate class (Caches, Queues, SharedState), each with bindings to appropriate namespaces within the KV store. * the modified CI launches 1 redis instance, and assigns each KV store to a different `database number` --- areas that need some more attention * [x] CI support * [x] Build Support - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Deployment (Omnibus) - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Developer Support (GDK) - deferred to issue gitlab-org/gitlab-development-kit#238 * [x] Documentation * [x] retention policy for different namespaces * [x] abstract class between new KV objects and Redis for shared behaviors * [x] Migration from single instance to multiple instances - deferred to issue gitlab-org/gitlab-ce#33185 - deferred to issue gitlab-org/gitlab-ce#33181 - ~~* [ ] using atomic transactions~~ - deferred to investigation issue gitlab-org/gitlab-ce#33182 - ~~* [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes~~ - deferred to release management process - ~~* [ ] Changelog~~ - deferred - ~~* [ ] using actual Model objects for encapsulation of underlying technology~~ - deferred - ~~* [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception~~ --- config/README.md | 58 ++++++++++++++++++++----------------- lib/gitlab/redis/wrapper.rb | 9 +++++- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/config/README.md b/config/README.md index 82dc9ae98b41..2778d0d4f02e 100644 --- a/config/README.md +++ b/config/README.md @@ -39,6 +39,18 @@ If desired, the routing URL provided by these settings can be used with: 1. `host name` or IP for each Redis instance desired 2. TCP port number for each Redis instance desired 3. `database number` for each Redis instance desired + +## Example URL attribute formats for GitLab Redis `.yml` configuration files +* Unix Socket, default Redis database (0) + * `url: unix:/path/to/redis.sock` + * `url: unix:/path/to/redis.sock?db=` +* Unix Socket, Redis database 44 + * `url: unix:/path/to/redis.sock?db=44` + * `url: unix:/path/to/redis.sock?extra=foo&db=44` +* TCP Socket for Redis on localhost, port 6379, database 33 + * `url: redis://:mynewpassword@localhost:6379/33` +* TCP Socket for Redis on remote host `myserver`, port 6379, database 33 + * `url: redis://:mynewpassword@myserver:6379/33` ## redis.cache.yml @@ -46,21 +58,19 @@ If configured, `redis.cache.yml` overrides the `resque.yml` settings to configure the Redis database instance used for `Rails.cache` and other volatile non-persistent data which enhances the performance of GitLab. -Settings here can be overridden by the environment variables -`DEFAULT_REDIS_CACHE_URL` which overrides the URL used by the Redis -connection manager, and `GITLAB_REDIS_CACHE_CONFIG_FILE` which provides +Settings here can be overridden by the environment variable +`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides an alternate location for configuration settings. The order of precedence for the URL used to connect to the Redis instance used for `cache` is: -1. URL from environment variable `DEFAULT_REDIS_CACHE_URL` -2. URL from a configuration file pointed to by the +1. URL from a configuration file pointed to by the `GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable -3. URL from `redis.cache.yml` -4. URL from environment variable `DEFAULT_REDIS_URL` -5. URL from a configuration file pointed to by the +2. URL from `redis.cache.yml` +3. URL from a configuration file pointed to by the `GITLAB_REDIS_CONFIG_FILE` environment variable -6. URL from `resque.yml` +4. URL from `resque.yml` +5. `redis://localhost:6380` The order of precedence for all other configuration settings for `cache` are selected from only the first of the following files found (if a setting @@ -82,21 +92,19 @@ These queues are intended to be the foundation of reliable inter-process communication between modules, whether on the same host node, or within a cluster. The primary clients of the queues are SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can -be overridden by the environment variables `DEFAULT_REDIS_QUEUES_URL` which - overrides the URL used by the Redis connection manager, and +be overridden by the environment variable `GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for configuration settings. The order of precedence for the URL used to connect to the Redis instance used for `queues` is: -1. URL from environment variable `DEFAULT_REDIS_QUEUES_URL` -2. URL from a configuration file pointed to by the +1. URL from a configuration file pointed to by the `GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable -3. URL from `redis.queues.yml` -4. URL from environment variable `DEFAULT_REDIS_URL` -5. URL from a configuration file pointed to by the +2. URL from `redis.queues.yml` +3. URL from a configuration file pointed to by the `GITLAB_REDIS_CONFIG_FILE` environment variable -6. URL from `resque.yml` +4. URL from `resque.yml` +5. `redis://localhost:6381` The order of precedence for all other configuration settings for `queues` are selected from only the first of the following files found (if a setting @@ -115,21 +123,19 @@ If configured, `redis.shared_state.yml` overrides the `resque.yml` settings to configure the Redis database instance used for clients of `::Gitlab::Redis::SharedState` such as session state, and rate limiting. -Settings here can be overridden by the environment variables -`DEFAULT_REDIS_SHARED_STATE_URL` which overrides the URL used by the Redis -connection manager, and `GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides +Settings here can be overridden by the environment variable +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides an alternate location for configuration settings. The order of precedence for the URL used to connect to the Redis instance used for `shared_state` is: -1. URL from environment variable `DEFAULT_REDIS_SHARED_STATE_URL` -2. URL from a configuration file pointed to by the +1. URL from a configuration file pointed to by the `GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable -3. URL from `redis.shared_state.yml` -4. URL from environment variable `DEFAULT_REDIS_URL` -5. URL from a configuration file pointed to by the +2. URL from `redis.shared_state.yml` +3. URL from a configuration file pointed to by the `GITLAB_REDIS_CONFIG_FILE` environment variable -6. URL from `resque.yml` +4. URL from `resque.yml` +5. `redis://localhost:6382` The order of precedence for all other configuration settings for `shared_state` are selected from only the first of the following files found (if a setting diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index b68c6347f371..6759fd8132f9 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -50,7 +50,7 @@ def config_file_name # otherwise, if config files exists for wrapper class, use it file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) - # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + # this will force use of DEFAULT_REDIS_URL when config file is absent file_name = nil unless File.file?(file_name) file_name end @@ -86,12 +86,19 @@ def redis_store_options if redis_uri.scheme == 'unix' # Redis::Store does not handle Unix sockets well, so let's do it for them config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.has_key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) # order is important here, sentinels must be after the connection keys. # {url: ..., port: ..., sentinels: [...]} redis_hash.merge(config) + pp redis_hash end end -- GitLab From 2552c2c82fad751ca2333c33d0e8b5b4bf410aef Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 2 Jun 2017 20:30:55 +0800 Subject: [PATCH 40/62] https://gitlab.com/gitlab-org/gitlab-ce/issues/30392 support for multiple Redis key value stores * Add support for multiple key-value stores * allows different policies for retention, backup, deployment topologies by key namespace * Abstract key-value store name away from name of underlying technology such as "Redis" * note that config files, unix port numbers, and unix sockets for communication with Redis have been cloned and reconfigured --- * this MR is primarily submitted for review of * design approach * scope of changes * size of MR (1000+ lines -- could be broken into a few smaller step-wise pieces) * GDK testing can be performed with [WIP MR for gdk](https://gitlab.com/gitlab-org/gitlab-development-kit/merge_requests/298) * the Gitlab::Redis class was deliberately removed/deprecated to force the runtime to find any uses that were not readily apparent from grep/search of git repository * the modified GDK launches 3 separate KV servers (redis instances at present moment) and wraps each instance in a separate class (Caches, Queues, SharedState), each with bindings to appropriate namespaces within the KV store. * the modified CI launches 1 redis instance, and assigns each KV store to a different `database number` --- areas that need some more attention * [x] CI support * [x] Build Support - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Deployment (Omnibus) - deferred to issue gitlab-org/omnibus-gitlab#2389 * [x] Developer Support (GDK) - deferred to issue gitlab-org/gitlab-development-kit#238 * [x] Documentation * [x] retention policy for different namespaces * [x] abstract class between new KV objects and Redis for shared behaviors * [x] Migration from single instance to multiple instances - deferred to issue gitlab-org/gitlab-ce#33185 - deferred to issue gitlab-org/gitlab-ce#33181 - ~~* [ ] using atomic transactions~~ - deferred to investigation issue gitlab-org/gitlab-ce#33182 - ~~* [ ] whether certain key-values can be local to one FE node, or MUST be shared with other FE nodes~~ - deferred to release management process - ~~* [ ] Changelog~~ - deferred - ~~* [ ] using actual Model objects for encapsulation of underlying technology~~ - deferred - ~~* [ ] support for key-value pairs to participate in Rails Transactions with rollback on exception~~ --- lib/gitlab/redis/wrapper.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 6759fd8132f9..6fbb6de61803 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -98,7 +98,6 @@ def redis_store_options # order is important here, sentinels must be after the connection keys. # {url: ..., port: ..., sentinels: [...]} redis_hash.merge(config) - pp redis_hash end end -- GitLab From fc6bbf7d931bc9b74ae5c8ba1b3d4cbeb867c58f Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sat, 3 Jun 2017 09:14:50 +0800 Subject: [PATCH 41/62] change path name for readability and maintainability --- lib/gitlab/redis/cache.rb | 2 +- lib/gitlab/redis/queues.rb | 2 +- lib/gitlab/redis/shared_state.rb | 2 +- lib/gitlab/redis/wrapper.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 54e7b64444fc..af5bd1e2b018 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -7,7 +7,7 @@ module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = '../../../config/redis.cache.yml'.freeze + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config','redis.cache.yml').freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze class << self diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 4200f6ef9d64..13a2f622452f 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,7 +8,7 @@ class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = '../../../config/redis.queues.yml'.freeze + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config','redis.queues.yml').freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze class << self diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index be7f191b213f..3c405c74b6af 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,7 +7,7 @@ module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = '../../../config/redis.shared_state.yml'.freeze + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config','redis.shared_state.yml').freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze class << self diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 6fbb6de61803..7c1d7f6e9498 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -10,7 +10,7 @@ module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - DEFAULT_REDIS_CONFIG_FILE_NAME = '../../../config/resque.yml'.freeze + DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config','resque.yml').freeze class << self delegate :params, :url, to: :new -- GitLab From 4413fdbb4fea79d50170890748d0c30e4c0a2fcd Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sat, 3 Jun 2017 17:03:58 +0800 Subject: [PATCH 42/62] allow redis config retrieval to work in non-rails environments --- lib/gitlab/redis/cache.rb | 5 +++-- lib/gitlab/redis/queues.rb | 5 +++-- lib/gitlab/redis/shared_state.rb | 5 +++-- lib/gitlab/redis/wrapper.rb | 4 +++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index af5bd1e2b018..6faf8f466e85 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,4 +1,3 @@ -# This file should not have any direct dependency on Rails environment # please require all dependencies below: require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) @@ -7,8 +6,10 @@ module Redis class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config','redis.cache.yml').freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + unless Rails.root.nil? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config','redis.cache.yml').freeze + end class << self delegate :params, :config_file_name, :url, to: :new diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 13a2f622452f..94220c12265a 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,4 +1,3 @@ -# This file should not have any direct dependency on Rails environment # please require all dependencies below: require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) @@ -8,8 +7,10 @@ class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config','redis.queues.yml').freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + unless Rails.root.nil? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config','redis.queues.yml').freeze + end class << self delegate :params, :config_file_name, :url, to: :new diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 3c405c74b6af..d075ecaf61c8 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,4 +1,3 @@ -# This file should not have any direct dependency on Rails environment # please require all dependencies below: require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) @@ -7,8 +6,10 @@ module Redis class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config','redis.shared_state.yml').freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + unless Rails.root.nil? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config','redis.shared_state.yml').freeze + end class << self delegate :params, :config_file_name, :url, to: :new diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 7c1d7f6e9498..a364fc853b33 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -10,7 +10,9 @@ module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config','resque.yml').freeze + unless Rails.root.nil? + DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config','resque.yml').freeze + end class << self delegate :params, :url, to: :new -- GitLab From 9e13976705f36abed494150dcb2a8bc64386069c Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sat, 3 Jun 2017 18:11:25 +0800 Subject: [PATCH 43/62] use shared_examples for redis configuration testing --- spec/lib/gitlab/redis/cache_spec.rb | 224 ++----------------- spec/lib/gitlab/redis/queues_spec.rb | 225 ++------------------ spec/lib/gitlab/redis/shared_state_spec.rb | 224 ++----------------- spec/lib/gitlab/redis/wrapper_spec.rb | 224 ++----------------- spec/support/redis/redis_shared_examples.rb | 202 ++++++++++++++++++ 5 files changed, 271 insertions(+), 828 deletions(-) create mode 100644 spec/support/redis/redis_shared_examples.rb diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb index 3f36e2164b5e..429077662c1d 100644 --- a/spec/lib/gitlab/redis/cache_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -1,211 +1,21 @@ require 'spec_helper' describe Gitlab::Redis::Cache do - include StubENV - - let(:config) { 'config/redis.cache.yml' } - - before(:each) do - stub_env('GITLAB_REDIS_CACHE_CONFIG_FILE', Rails.root.join(config).to_s) - clear_raw_config - end - - after(:each) do - clear_raw_config - end - - describe '.params' do - subject { described_class.params } - - it 'withstands mutation' do - params1 = described_class.params - params2 = described_class.params - params1[:foo] = :bar - - expect(params2).not_to have_key(:foo) - end - - context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_socket.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.cache.sock') - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.cache.sock') - is_expected.not_to have_key(:url) - end - end - end - - context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6380, db: 99) - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6380, db: 99) - is_expected.not_to have_key(:url) - end - end - end - end - - describe '.url' do - it 'withstands mutation' do - url1 = described_class.url - url2 = described_class.url - url1 << 'foobar' - - expect(url2).not_to end_with('foobar') - end - - context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_cache_config_with_env.yml' } - - before do - stub_env('TEST_GITLAB_REDIS_CACHE_URL', 'redis://redishost:6380') - end - - it 'reads caching url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6380' - end - end - end - - describe '._raw_config' do - subject { described_class._raw_config } - let(:config) { '/var/empty/doesnotexist' } - - it 'should be frozen' do - expect(subject).to be_frozen - end - - it 'returns false when the file does not exist' do - expect(subject).to eq(false) - end - end - - describe '.with' do - before { clear_pool } - after { clear_pool } - - context 'when running not on sidekiq workers' do - before { allow(Sidekiq).to receive(:server?).and_return(false) } - - it 'instantiates a connection pool with size 5' do - expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - - described_class.with { |_caching| true } - end - end - - context 'when running on sidekiq workers' do - before do - allow(Sidekiq).to receive(:server?).and_return(true) - allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) - end - - it 'instantiates a connection pool with a size based on the concurrency of the worker' do - expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - - described_class.with { |_caching| true } - end - end - end - - describe '#sentinels' do - subject { described_class.new(Rails.env).sentinels } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } - - it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26380) - is_expected.to include(host: 'slave2', port: 26380) - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } - - it 'returns nil' do - is_expected.to be_nil - end - end - end - - describe '#sentinels?' do - subject { described_class.new(Rails.env).sentinels? } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_cache_new_format_host.yml' } - - it 'returns true' do - is_expected.to be_truthy - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_cache_old_format_host.yml' } - - it 'returns false' do - is_expected.to be_falsey - end - end - end - - describe '#raw_config_hash' do - it 'returns default caching url when no config file is present' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL) - end - - it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6380' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6380') - end - end - - describe '#fetch_config' do - it 'returns false when no config file is present' do - allow(described_class).to receive(:_raw_config) { false } - - expect(subject.send(:fetch_config)).to be_falsey - end - end - - def clear_raw_config - described_class.remove_instance_variable(:@_raw_config) - rescue NameError - # raised if @_raw_config was not set; ignore - end - - def clear_pool - described_class.remove_instance_variable(:@pool) - rescue NameError - # raised if @pool was not set; ignore - end + let(:config_file_name) { "config/redis.cache.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_CACHE_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_cache_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_cache_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.cache.sock" } + let(:new_socket_path) {"/path/to/redis.cache.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_cache_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_cache_new_format_host.yml" } + let(:redis_port) { 6380 } + let(:redis_database) { 99 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_cache_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_CACHE_URL"} + let(:test_redis_url) { "redis://redishost:#{redis_port}"} + let(:class_redis_url) { Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL } + + include_examples "redis_shared_examples" end diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index d4878349edd1..ddc82a32aee4 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -1,211 +1,22 @@ require 'spec_helper' describe Gitlab::Redis::Queues do - include StubENV - - let(:config) { 'config/redis.queues.yml' } - - before(:each) do - stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', Rails.root.join(config).to_s) - clear_raw_config - end - - after(:each) do - clear_raw_config - end - - describe '.params' do - subject { described_class.params } - - it 'withstands mutation' do - params1 = described_class.params - params2 = described_class.params - params1[:foo] = :bar - - expect(params2).not_to have_key(:foo) - end - - context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_queues_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_queues_new_format_socket.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.queues.sock') - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.queues.sock') - is_expected.not_to have_key(:url) - end - end - end - - context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6381, db: 99) - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6381, db: 99) - is_expected.not_to have_key(:url) - end - end - end - end - - describe '.url' do - it 'withstands mutation' do - url1 = described_class.url - url2 = described_class.url - url1 << 'foobar' - - expect(url2).not_to end_with('foobar') - end - - context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_queues_config_with_env.yml' } - - before do - stub_env('TEST_GITLAB_REDIS_QUEUES_URL', 'redis://redishost:6381') - end - - it 'reads queues url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6381' - end - end - end - - describe '._raw_config' do - subject { described_class._raw_config } - let(:config) { '/var/empty/doesnotexist' } - - it 'should be frozen' do - expect(subject).to be_frozen - end - - it 'returns false when the file does not exist' do - expect(subject).to eq(false) - end - end - - describe '.with' do - before { clear_pool } - after { clear_pool } - - context 'when running not on sidekiq workers' do - before { allow(Sidekiq).to receive(:server?).and_return(false) } - - it 'instantiates a connection pool with size 5' do - expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - - described_class.with { |_queues| true } - end - end - - context 'when running on sidekiq workers' do - before do - allow(Sidekiq).to receive(:server?).and_return(true) - allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) - end - - it 'instantiates a connection pool with a size based on the concurrency of the worker' do - expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - - described_class.with { |_queues| true } - end - end - end - - describe '#sentinels' do - subject { described_class.new(Rails.env).sentinels } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - - it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26381) - is_expected.to include(host: 'slave2', port: 26381) - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } - - it 'returns nil' do - is_expected.to be_nil - end - end - end - - describe '#sentinels?' do - subject { described_class.new(Rails.env).sentinels? } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - - it 'returns true' do - is_expected.to be_truthy - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_queues_old_format_host.yml' } - - it 'returns false' do - is_expected.to be_falsey - end - end - end - - describe '#raw_config_hash' do - it 'returns default queues url when no config file is present' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL) - end - - it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6381' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6381') - end - end - - describe '#fetch_config' do - it 'returns false when no config file is present' do - allow(described_class).to receive(:_raw_config) { false } - - expect(subject.send(:fetch_config)).to be_falsey - end - end - - def clear_raw_config - described_class.remove_instance_variable(:@_raw_config) - rescue NameError - # raised if @_raw_config was not set; ignore - end - - def clear_pool - described_class.remove_instance_variable(:@pool) - rescue NameError - # raised if @pool was not set; ignore - end + let(:config_file_name) { "config/redis.queues.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_QUEUES_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_queues_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_queues_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.queues.sock" } + let(:new_socket_path) {"/path/to/redis.queues.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_queues_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_queues_new_format_host.yml" } + let(:redis_port) { 6381 } + let(:redis_database) { 99 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_queues_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_QUEUES_URL"} + let(:test_redis_url) { "redis://redishost:#{redis_port}"} + let(:class_redis_url) { Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL } + + include_examples "redis_shared_examples" end + diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb index 978b6c6096ef..1aa2bf3b47c9 100644 --- a/spec/lib/gitlab/redis/shared_state_spec.rb +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -1,211 +1,21 @@ require 'spec_helper' describe Gitlab::Redis::SharedState do - include StubENV - - let(:config) { 'config/redis.shared_state.yml' } - - before(:each) do - stub_env('GITLAB_REDIS_SHARED_STATE_CONFIG_FILE', Rails.root.join(config).to_s) - clear_raw_config - end - - after(:each) do - clear_raw_config - end - - describe '.params' do - subject { described_class.params } - - it 'withstands mutation' do - params1 = described_class.params - params2 = described_class.params - params1[:foo] = :bar - - expect(params2).not_to have_key(:foo) - end - - context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_shared_state_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_shared_state_new_format_socket.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.shared_state.sock') - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.shared_state.sock') - is_expected.not_to have_key(:url) - end - end - end - - context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6382, db: 99) - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6382, db: 99) - is_expected.not_to have_key(:url) - end - end - end - end - - describe '.url' do - it 'withstands mutation' do - url1 = described_class.url - url2 = described_class.url - url1 << 'foobar' - - expect(url2).not_to end_with('foobar') - end - - context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_shared_state_config_with_env.yml' } - - before do - stub_env('TEST_GITLAB_REDIS_SHARED_STATE_URL', 'redis://redishost:6382') - end - - it 'reads redis_shared_state url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6382' - end - end - end - - describe '._raw_config' do - subject { described_class._raw_config } - let(:config) { '/var/empty/doesnotexist' } - - it 'should be frozen' do - expect(subject).to be_frozen - end - - it 'returns false when the file does not exist' do - expect(subject).to eq(false) - end - end - - describe '.with' do - before { clear_pool } - after { clear_pool } - - context 'when running not on sidekiq workers' do - before { allow(Sidekiq).to receive(:server?).and_return(false) } - - it 'instantiates a connection pool with size 5' do - expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - - described_class.with { |_redis_shared_state| true } - end - end - - context 'when running on sidekiq workers' do - before do - allow(Sidekiq).to receive(:server?).and_return(true) - allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) - end - - it 'instantiates a connection pool with a size based on the concurrency of the worker' do - expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - - described_class.with { |_redis_shared_state| true } - end - end - end - - describe '#sentinels' do - subject { described_class.new(Rails.env).sentinels } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } - - it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26382) - is_expected.to include(host: 'slave2', port: 26382) - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } - - it 'returns nil' do - is_expected.to be_nil - end - end - end - - describe '#sentinels?' do - subject { described_class.new(Rails.env).sentinels? } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_shared_state_new_format_host.yml' } - - it 'returns true' do - is_expected.to be_truthy - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_shared_state_old_format_host.yml' } - - it 'returns false' do - is_expected.to be_falsey - end - end - end - - describe '#raw_config_hash' do - it 'returns default redis_shared_state url when no config file is present' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL) - end - - it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6382' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6382') - end - end - - describe '#fetch_config' do - it 'returns false when no config file is present' do - allow(described_class).to receive(:_raw_config) { false } - - expect(subject.send(:fetch_config)).to be_falsey - end - end - - def clear_raw_config - described_class.remove_instance_variable(:@_raw_config) - rescue NameError - # raised if @_raw_config was not set; ignore - end - - def clear_pool - described_class.remove_instance_variable(:@pool) - rescue NameError - # raised if @pool was not set; ignore - end + let(:config_file_name) { "config/redis.shared_state.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_shared_state_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_shared_state_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.shared_state.sock" } + let(:new_socket_path) {"/path/to/redis.shared_state.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_shared_state_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_shared_state_new_format_host.yml" } + let(:redis_port) { 6382 } + let(:redis_database) { 99 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_shared_state_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_SHARED_STATE_URL"} + let(:test_redis_url) { "redis://redishost:#{redis_port}"} + let(:class_redis_url) { Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL } + + include_examples "redis_shared_examples" end diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb index 214361c6ece9..44b73140db81 100644 --- a/spec/lib/gitlab/redis/wrapper_spec.rb +++ b/spec/lib/gitlab/redis/wrapper_spec.rb @@ -1,211 +1,21 @@ require 'spec_helper' describe Gitlab::Redis::Wrapper do - include StubENV - - let(:config) { 'config/resque.yml' } - - before(:each) do - stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s) - clear_raw_config - end - - after(:each) do - clear_raw_config - end - - describe '.params' do - subject { described_class.params } - - it 'withstands mutation' do - params1 = described_class.params - params2 = described_class.params - params1[:foo] = :bar - - expect(params2).not_to have_key(:foo) - end - - context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.sock') - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.sock') - is_expected.not_to have_key(:url) - end - end - end - - context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' } - - context 'with old format' do - let(:config) { config_old } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) - is_expected.not_to have_key(:url) - end - end - - context 'with new format' do - let(:config) { config_new } - - it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) - is_expected.not_to have_key(:url) - end - end - end - end - - describe '.url' do - it 'withstands mutation' do - url1 = described_class.url - url2 = described_class.url - url1 << 'foobar' - - expect(url2).not_to end_with('foobar') - end - - context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' } - - before do - stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379') - end - - it 'reads redis url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6379' - end - end - end - - describe '._raw_config' do - subject { described_class._raw_config } - let(:config) { '/var/empty/doesnotexist' } - - it 'should be frozen' do - expect(subject).to be_frozen - end - - it 'returns false when the file does not exist' do - expect(subject).to eq(false) - end - end - - describe '.with' do - before { clear_pool } - after { clear_pool } - - context 'when running not on sidekiq workers' do - before { allow(Sidekiq).to receive(:server?).and_return(false) } - - it 'instantiates a connection pool with size 5' do - expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - - described_class.with { |_wrapper| true } - end - end - - context 'when running on sidekiq workers' do - before do - allow(Sidekiq).to receive(:server?).and_return(true) - allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) - end - - it 'instantiates a connection pool with a size based on the concurrency of the worker' do - expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - - described_class.with { |_wrapper| true } - end - end - end - - describe '#sentinels' do - subject { described_class.new(Rails.env).sentinels } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } - - it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26379) - is_expected.to include(host: 'slave2', port: 26379) - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } - - it 'returns nil' do - is_expected.to be_nil - end - end - end - - describe '#sentinels?' do - subject { described_class.new(Rails.env).sentinels? } - - context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } - - it 'returns true' do - is_expected.to be_truthy - end - end - - context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } - - it 'returns false' do - is_expected.to be_falsey - end - end - end - - describe '#raw_config_hash' do - it 'returns default redis url when no config file is present' do - expect(subject).to receive(:fetch_config) { false } - - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL) - end - - it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') - end - end - - describe '#fetch_config' do - it 'returns false when no config file is present' do - allow(described_class).to receive(:_raw_config) { false } - - expect(subject.send(:fetch_config)).to be_falsey - end - end - - def clear_raw_config - described_class.remove_instance_variable(:@_raw_config) - rescue NameError - # raised if @_raw_config was not set; ignore - end - - def clear_pool - described_class.remove_instance_variable(:@pool) - rescue NameError - # raised if @pool was not set; ignore - end + let(:config_file_name) { "config/resque.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.sock" } + let(:new_socket_path) {"/path/to/redis.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" } + let(:redis_port) { 6379 } + let(:redis_database) { 99 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"} + let(:test_redis_url) { "redis://redishost:#{redis_port}"} + let(:class_redis_url) { Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL } + + include_examples "redis_shared_examples" end diff --git a/spec/support/redis/redis_shared_examples.rb b/spec/support/redis/redis_shared_examples.rb new file mode 100644 index 000000000000..d0cc85b84fb0 --- /dev/null +++ b/spec/support/redis/redis_shared_examples.rb @@ -0,0 +1,202 @@ +RSpec.shared_examples "redis_shared_examples" do + include StubENV + + before(:each) do + stub_env(environment_config_file_name, Rails.root.join(config_file_name)) + clear_raw_config + end + + after(:each) do + clear_raw_config + end + + describe '.params' do + subject { described_class.params } + + it 'withstands mutation' do + params1 = described_class.params + params2 = described_class.params + params1[:foo] = :bar + + expect(params2).not_to have_key(:foo) + end + + context 'when url contains unix socket reference' do + context 'with old format' do + let(:config_file_name) { config_old_format_socket } + + it 'returns path key instead' do + is_expected.to include(path: old_socket_path) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config_file_name) { config_new_format_socket } + + it 'returns path key instead' do + is_expected.to include(path: new_socket_path) + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + + context 'with old format' do + let(:config_file_name) { config_old_format_host } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mypassword', port: redis_port, db: redis_database) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + let(:config_file_name) { config_new_format_host } + + it 'returns hash with host, port, db, and password' do + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: redis_port, db: redis_database) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '.url' do + it 'withstands mutation' do + url1 = described_class.url + url2 = described_class.url + url1 << 'foobar' + + expect(url2).not_to end_with('foobar') + end + + context 'when yml file with env variable' do + let(:config_file_name) { config_with_environment_variable_inside } + + before do + stub_env(config_env_variable_url, test_redis_url) + end + + it 'reads redis url from env variable' do + expect(described_class.url).to eq test_redis_url + end + end + end + + describe '._raw_config' do + subject { described_class._raw_config } + let(:config_file_name) { '/var/empty/doesnotexist' } + + it 'should be frozen' do + expect(subject).to be_frozen + end + + it 'returns false when the file does not exist' do + expect(subject).to eq(false) + end + end + + describe '.with' do + before { clear_pool } + after { clear_pool } + + context 'when running not on sidekiq workers' do + before { allow(Sidekiq).to receive(:server?).and_return(false) } + + it 'instantiates a connection pool with size 5' do + expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original + + described_class.with { |_redis_shared_example| true } + end + end + + context 'when running on sidekiq workers' do + before do + allow(Sidekiq).to receive(:server?).and_return(true) + allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 }) + end + + it 'instantiates a connection pool with a size based on the concurrency of the worker' do + expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original + + described_class.with { |_redis_shared_example| true } + end + end + end + + describe '#sentinels' do + subject { described_class.new(Rails.env).sentinels } + + context 'when sentinels are defined' do + let(:config_file_name) { config_new_format_host } + + it 'returns an array of hashes with host and port keys' do + is_expected.to include(host: 'localhost', port: sentinel_port) + is_expected.to include(host: 'slave2', port: sentinel_port) + end + end + + context 'when sentinels are not defined' do + let(:config_file_name) { config_old_format_host } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#sentinels?' do + subject { described_class.new(Rails.env).sentinels? } + + context 'when sentinels are defined' do + let(:config_file_name) { config_new_format_host } + + it 'returns true' do + is_expected.to be_truthy + end + end + + context 'when sentinels are not defined' do + let(:config_file_name) { config_old_format_host } + + it 'returns false' do + is_expected.to be_falsey + end + end + end + + describe '#raw_config_hash' do + it 'returns default redis url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: class_redis_url ) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { test_redis_url } + expect(subject.send(:raw_config_hash)).to eq(url: test_redis_url) + end + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(described_class).to receive(:_raw_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end + + def clear_raw_config + described_class.remove_instance_variable(:@_raw_config) + rescue NameError + # raised if @_raw_config was not set; ignore + end + + def clear_pool + described_class.remove_instance_variable(:@pool) + rescue NameError + # raised if @pool was not set; ignore + end +end -- GitLab From 5554ac0be1c5a1b635dd17cc455f6a442031f460 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Sat, 3 Jun 2017 19:24:21 +0800 Subject: [PATCH 44/62] clean rubocop style issues --- lib/gitlab/redis/cache.rb | 2 +- lib/gitlab/redis/queues.rb | 2 +- lib/gitlab/redis/shared_state.rb | 2 +- lib/gitlab/redis/wrapper.rb | 2 +- spec/lib/gitlab/redis/queues_spec.rb | 1 - spec/support/redis/redis_shared_examples.rb | 1 - 6 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 6faf8f466e85..00807662516b 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -8,7 +8,7 @@ class Cache < ::Gitlab::Redis::Wrapper DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze unless Rails.root.nil? - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config','redis.cache.yml').freeze + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.cache.yml').freeze end class << self diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 94220c12265a..c891799a80ac 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -9,7 +9,7 @@ class Queues < ::Gitlab::Redis::Wrapper DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze unless Rails.root.nil? - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config','redis.queues.yml').freeze + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.queues.yml').freeze end class << self diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index d075ecaf61c8..1621034056bf 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -8,7 +8,7 @@ class SharedState < ::Gitlab::Redis::Wrapper DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze unless Rails.root.nil? - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config','redis.shared_state.yml').freeze + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.shared_state.yml').freeze end class << self diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index a364fc853b33..1c99809972bb 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -11,7 +11,7 @@ class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze unless Rails.root.nil? - DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config','resque.yml').freeze + DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config', 'resque.yml').freeze end class << self diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index ddc82a32aee4..8f2418fad414 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -19,4 +19,3 @@ include_examples "redis_shared_examples" end - diff --git a/spec/support/redis/redis_shared_examples.rb b/spec/support/redis/redis_shared_examples.rb index d0cc85b84fb0..564186ba9baf 100644 --- a/spec/support/redis/redis_shared_examples.rb +++ b/spec/support/redis/redis_shared_examples.rb @@ -42,7 +42,6 @@ end context 'when url is host based' do - context 'with old format' do let(:config_file_name) { config_old_format_host } -- GitLab From 68c622c64cb53b372f695869366dc4e7dce659ed Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 7 Jun 2017 10:44:25 +0800 Subject: [PATCH 45/62] update rubocop flagged items --- lib/gitlab/redis/wrapper.rb | 2 +- spec/requests/api/users_spec.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 1c99809972bb..1fa55ec2d960 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -91,7 +91,7 @@ def redis_store_options query = redis_uri.query unless query.nil? queries = CGI.parse(redis_uri.query) - db_numbers = queries["db"] if queries.has_key?("db") + db_numbers = queries["db"] if queries.key?("db") config[:db] = db_numbers[0].to_i if db_numbers.any? end config diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index 353d05d514be..8f62af34d331 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1130,7 +1130,6 @@ end end - context "user activities", :clean_gitlab_redis_shared_state do let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } -- GitLab From 05f94aadf4f360042e4111e6f85c117dfd6a1fd7 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 8 Jun 2017 12:40:39 +0800 Subject: [PATCH 46/62] update for code drift in the rebase --- app/services/metrics_service.rb | 5 ++++- spec/controllers/metrics_controller_spec.rb | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/services/metrics_service.rb b/app/services/metrics_service.rb index d726db4e99bc..c92f070601cd 100644 --- a/app/services/metrics_service.rb +++ b/app/services/metrics_service.rb @@ -3,7 +3,10 @@ class MetricsService CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCheck, + Gitlab::HealthChecks::Redis::RedisCheck, + Gitlab::HealthChecks::Redis::CacheCheck, + Gitlab::HealthChecks::Redis::QueuesCheck, + Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/spec/controllers/metrics_controller_spec.rb b/spec/controllers/metrics_controller_spec.rb index e4640843eb56..f7942166618f 100644 --- a/spec/controllers/metrics_controller_spec.rb +++ b/spec/controllers/metrics_controller_spec.rb @@ -36,21 +36,23 @@ end it 'returns Caching ping metrics' do - get :metrics + get :index expect(response.body).to match(/^redis_cache_ping_timeout 0$/) expect(response.body).to match(/^redis_cache_ping_success 1$/) expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/) end it 'returns Queues ping metrics' do - get :metrics + get :index + expect(response.body).to match(/^redis_queues_ping_timeout 0$/) expect(response.body).to match(/^redis_queues_ping_success 1$/) expect(response.body).to match(/^redis_queues_ping_latency [0-9\.]+$/) end it 'returns SharedState ping metrics' do - get :metrics + get :index + expect(response.body).to match(/^redis_shared_state_ping_timeout 0$/) expect(response.body).to match(/^redis_shared_state_ping_success 1$/) expect(response.body).to match(/^redis_shared_state_ping_latency [0-9\.]+$/) -- GitLab From d4dbf6bb70fc69c2914e9959e02208ff24827415 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 9 Jun 2017 06:24:44 +0800 Subject: [PATCH 47/62] updated per merge review discussions --- .../operations/cleaning_up_redis_sessions.md | 2 +- lib/gitlab/health_checks/redis/redis_check.rb | 4 +-- lib/gitlab/redis/cache.rb | 28 +++++++++---------- lib/gitlab/redis/queues.rb | 28 +++++++++---------- lib/gitlab/redis/shared_state.rb | 28 +++++++++---------- lib/gitlab/redis/wrapper.rb | 10 +++---- lib/tasks/cache.rake | 24 ++++++++-------- 7 files changed, 62 insertions(+), 62 deletions(-) diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 3b7e9b1a5036..3a35aff83667 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -15,7 +15,7 @@ prefixed with 'session:gitlab:', so they would look like 'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to remove the keys in the old format. -**Note :** the instructions below must be modified in accordance with your +**Note:** the instructions below must be modified in accordance with your configuration settings if you have used the advanced Redis settings outlined in [Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md). diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index 7bdb9e5ec5ac..fe4e3c4a3ab5 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -16,8 +16,8 @@ def is_successful?(result) end def check - ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && \ - ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && \ + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 00807662516b..c7b7ce1be0fb 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -12,24 +12,24 @@ class Cache < ::Gitlab::Redis::Wrapper end class << self - delegate :params, :config_file_name, :url, to: :new - end + delegate :params, :url, to: :new - def default_url - DEFAULT_REDIS_CACHE_URL - end + def default_url + DEFAULT_REDIS_CACHE_URL + end - def config_file_name - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] - return file_name unless file_name.nil? + def config_file_name + # otherwise, if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? - # otherwise, if config files exists for wrapper class, use it - file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) - # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent - file_name = super unless File.file?(file_name) - file_name + # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent + file_name = super unless File.file?(file_name) + file_name + end end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index c891799a80ac..3726f64de549 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -13,24 +13,24 @@ class Queues < ::Gitlab::Redis::Wrapper end class << self - delegate :params, :config_file_name, :url, to: :new - end + delegate :params, :url, to: :new - def default_url - DEFAULT_REDIS_QUEUES_URL - end + def default_url + DEFAULT_REDIS_QUEUES_URL + end - def config_file_name - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] - return file_name if file_name + def config_file_name + # otherwise, if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name - # otherwise, if config files exists for wrapper class, use it - file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent - file_name = super unless File.file?(file_name) - file_name + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 1621034056bf..5c4f8a9817d4 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -12,24 +12,24 @@ class SharedState < ::Gitlab::Redis::Wrapper end class << self - delegate :params, :config_file_name, :url, to: :new - end + delegate :params, :url, to: :new - def default_url - DEFAULT_REDIS_SHARED_STATE_URL - end + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end - def config_file_name - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist - file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] - return file_name if file_name + def config_file_name + # otherwise, if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name - # otherwise, if config files exists for wrapper class, use it - file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent - file_name = super unless File.file?(file_name) - file_name + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 1fa55ec2d960..afb2674ed6bc 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -44,6 +44,10 @@ def _raw_config @_raw_config end + def default_url + DEFAULT_REDIS_URL + end + def config_file_name # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] @@ -103,17 +107,13 @@ def redis_store_options end end - def default_url - DEFAULT_REDIS_URL - end - def raw_config_hash config_data = fetch_config if config_data config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys else - { url: default_url } + { url: self.class.default_url } end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index e063a943ebf3..564aa1419521 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,28 +1,28 @@ namespace :cache do namespace :clear do - KV_CACHE_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - KV_CACHE_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear key/value cache" - task gitlab_kv_cache: :environment do - Gitlab::Redis::Cache.with do |kv_cache| - cursor = KV_CACHE_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis::Cache.with do |redis| + cursor = REDIS_SCAN_START_STOP loop do - cursor, keys = kv_cache.scan( + cursor, keys = redis.scan( cursor, match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", - count: KV_CACHE_CLEAR_BATCH_SIZE + count: REDIS_CLEAR_BATCH_SIZE ) - kv_cache.del(*keys) if keys.any? + redis.del(*keys) if keys.any? - break if cursor == KV_CACHE_SCAN_START_STOP + break if cursor == REDIS_SCAN_START_STOP end end end - task all: [:gitlab_kv_cache] + task all: [:redis] end - task clear: 'cache:clear:gitlab_kv_cache' + task clear: 'cache:clear:redis' end -- GitLab From 2aad155b44054f8ba45d13bb0577cbdf7c608674 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 9 Jun 2017 06:45:45 +0800 Subject: [PATCH 48/62] updated per discussion on merge request --- spec/fixtures/config/redis_cache_new_format_host.yml | 6 +++--- spec/fixtures/config/redis_cache_old_format_host.yml | 6 +++--- spec/fixtures/config/redis_queues_new_format_host.yml | 6 +++--- spec/fixtures/config/redis_queues_old_format_host.yml | 6 +++--- spec/fixtures/config/redis_shared_state_new_format_host.yml | 6 +++--- spec/fixtures/config/redis_shared_state_old_format_host.yml | 6 +++--- spec/lib/gitlab/redis/cache_spec.rb | 3 +-- spec/lib/gitlab/redis/queues_spec.rb | 3 +-- spec/lib/gitlab/redis/shared_state_spec.rb | 3 +-- spec/lib/gitlab/redis/wrapper_spec.rb | 1 - spec/support/redis/redis_shared_examples.rb | 2 ++ 11 files changed, 23 insertions(+), 25 deletions(-) diff --git a/spec/fixtures/config/redis_cache_new_format_host.yml b/spec/fixtures/config/redis_cache_new_format_host.yml index 902bfa9ca148..a24f3716391d 100644 --- a/spec/fixtures/config/redis_cache_new_format_host.yml +++ b/spec/fixtures/config/redis_cache_new_format_host.yml @@ -1,7 +1,7 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis development: - url: redis://:mynewpassword@localhost:6380/99 + url: redis://:mynewpassword@localhost:6380/10 sentinels: - host: localhost @@ -10,7 +10,7 @@ development: host: slave2 port: 26380 # point to sentinel, not to redis port test: - url: redis://:mynewpassword@localhost:6380/99 + url: redis://:mynewpassword@localhost:6380/10 sentinels: - host: localhost @@ -19,7 +19,7 @@ test: host: slave2 port: 26380 # point to sentinel, not to redis port production: - url: redis://:mynewpassword@localhost:6380/99 + url: redis://:mynewpassword@localhost:6380/10 sentinels: - host: slave1 diff --git a/spec/fixtures/config/redis_cache_old_format_host.yml b/spec/fixtures/config/redis_cache_old_format_host.yml index 7164e088e56d..3609dcd022e5 100644 --- a/spec/fixtures/config/redis_cache_old_format_host.yml +++ b/spec/fixtures/config/redis_cache_old_format_host.yml @@ -1,5 +1,5 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis -development: redis://:mypassword@localhost:6380/99 -test: redis://:mypassword@localhost:6380/99 -production: redis://:mypassword@localhost:6380/99 +development: redis://:mypassword@localhost:6380/10 +test: redis://:mypassword@localhost:6380/10 +production: redis://:mypassword@localhost:6380/10 diff --git a/spec/fixtures/config/redis_queues_new_format_host.yml b/spec/fixtures/config/redis_queues_new_format_host.yml index d3e99d277df9..1535584d7799 100644 --- a/spec/fixtures/config/redis_queues_new_format_host.yml +++ b/spec/fixtures/config/redis_queues_new_format_host.yml @@ -1,7 +1,7 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis development: - url: redis://:mynewpassword@localhost:6381/99 + url: redis://:mynewpassword@localhost:6381/11 sentinels: - host: localhost @@ -10,7 +10,7 @@ development: host: slave2 port: 26381 # point to sentinel, not to redis port test: - url: redis://:mynewpassword@localhost:6381/99 + url: redis://:mynewpassword@localhost:6381/11 sentinels: - host: localhost @@ -19,7 +19,7 @@ test: host: slave2 port: 26381 # point to sentinel, not to redis port production: - url: redis://:mynewpassword@localhost:6381/99 + url: redis://:mynewpassword@localhost:6381/11 sentinels: - host: slave1 diff --git a/spec/fixtures/config/redis_queues_old_format_host.yml b/spec/fixtures/config/redis_queues_old_format_host.yml index 2dbc4c35d465..6531748a8d78 100644 --- a/spec/fixtures/config/redis_queues_old_format_host.yml +++ b/spec/fixtures/config/redis_queues_old_format_host.yml @@ -1,5 +1,5 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis -development: redis://:mypassword@localhost:6381/99 -test: redis://:mypassword@localhost:6381/99 -production: redis://:mypassword@localhost:6381/99 +development: redis://:mypassword@localhost:6381/11 +test: redis://:mypassword@localhost:6381/11 +production: redis://:mypassword@localhost:6381/11 diff --git a/spec/fixtures/config/redis_shared_state_new_format_host.yml b/spec/fixtures/config/redis_shared_state_new_format_host.yml index 2aea4fbf3e98..1180b2b4a820 100644 --- a/spec/fixtures/config/redis_shared_state_new_format_host.yml +++ b/spec/fixtures/config/redis_shared_state_new_format_host.yml @@ -1,7 +1,7 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis development: - url: redis://:mynewpassword@localhost:6382/99 + url: redis://:mynewpassword@localhost:6382/12 sentinels: - host: localhost @@ -10,7 +10,7 @@ development: host: slave2 port: 26382 # point to sentinel, not to redis port test: - url: redis://:mynewpassword@localhost:6382/99 + url: redis://:mynewpassword@localhost:6382/12 sentinels: - host: localhost @@ -19,7 +19,7 @@ test: host: slave2 port: 26382 # point to sentinel, not to redis port production: - url: redis://:mynewpassword@localhost:6382/99 + url: redis://:mynewpassword@localhost:6382/12 sentinels: - host: slave1 diff --git a/spec/fixtures/config/redis_shared_state_old_format_host.yml b/spec/fixtures/config/redis_shared_state_old_format_host.yml index cc922b0c2863..fef5e768c5db 100644 --- a/spec/fixtures/config/redis_shared_state_old_format_host.yml +++ b/spec/fixtures/config/redis_shared_state_old_format_host.yml @@ -1,5 +1,5 @@ # redis://[:password@]host[:port][/db-number][?option=value] # more details: http://www.iana.org/assignments/uri-schemes/prov/redis -development: redis://:mypassword@localhost:6382/99 -test: redis://:mypassword@localhost:6382/99 -production: redis://:mypassword@localhost:6382/99 +development: redis://:mypassword@localhost:6382/12 +test: redis://:mypassword@localhost:6382/12 +production: redis://:mypassword@localhost:6382/12 diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb index 429077662c1d..5a4f17cfcf6a 100644 --- a/spec/lib/gitlab/redis/cache_spec.rb +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -10,11 +10,10 @@ let(:config_old_format_host) { "spec/fixtures/config/redis_cache_old_format_host.yml" } let(:config_new_format_host) { "spec/fixtures/config/redis_cache_new_format_host.yml" } let(:redis_port) { 6380 } - let(:redis_database) { 99 } + let(:redis_database) { 10 } let(:sentinel_port) { redis_port + 20000 } let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_cache_config_with_env.yml"} let(:config_env_variable_url) {"TEST_GITLAB_REDIS_CACHE_URL"} - let(:test_redis_url) { "redis://redishost:#{redis_port}"} let(:class_redis_url) { Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL } include_examples "redis_shared_examples" diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb index 8f2418fad414..01ca25635a98 100644 --- a/spec/lib/gitlab/redis/queues_spec.rb +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -10,11 +10,10 @@ let(:config_old_format_host) { "spec/fixtures/config/redis_queues_old_format_host.yml" } let(:config_new_format_host) { "spec/fixtures/config/redis_queues_new_format_host.yml" } let(:redis_port) { 6381 } - let(:redis_database) { 99 } + let(:redis_database) { 11 } let(:sentinel_port) { redis_port + 20000 } let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_queues_config_with_env.yml"} let(:config_env_variable_url) {"TEST_GITLAB_REDIS_QUEUES_URL"} - let(:test_redis_url) { "redis://redishost:#{redis_port}"} let(:class_redis_url) { Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL } include_examples "redis_shared_examples" diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb index 1aa2bf3b47c9..24b73745dc52 100644 --- a/spec/lib/gitlab/redis/shared_state_spec.rb +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -10,11 +10,10 @@ let(:config_old_format_host) { "spec/fixtures/config/redis_shared_state_old_format_host.yml" } let(:config_new_format_host) { "spec/fixtures/config/redis_shared_state_new_format_host.yml" } let(:redis_port) { 6382 } - let(:redis_database) { 99 } + let(:redis_database) { 12 } let(:sentinel_port) { redis_port + 20000 } let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_shared_state_config_with_env.yml"} let(:config_env_variable_url) {"TEST_GITLAB_REDIS_SHARED_STATE_URL"} - let(:test_redis_url) { "redis://redishost:#{redis_port}"} let(:class_redis_url) { Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL } include_examples "redis_shared_examples" diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb index 44b73140db81..e1becd0a6145 100644 --- a/spec/lib/gitlab/redis/wrapper_spec.rb +++ b/spec/lib/gitlab/redis/wrapper_spec.rb @@ -14,7 +14,6 @@ let(:sentinel_port) { redis_port + 20000 } let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"} let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"} - let(:test_redis_url) { "redis://redishost:#{redis_port}"} let(:class_redis_url) { Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL } include_examples "redis_shared_examples" diff --git a/spec/support/redis/redis_shared_examples.rb b/spec/support/redis/redis_shared_examples.rb index 564186ba9baf..b2907e426f5e 100644 --- a/spec/support/redis/redis_shared_examples.rb +++ b/spec/support/redis/redis_shared_examples.rb @@ -1,6 +1,8 @@ RSpec.shared_examples "redis_shared_examples" do include StubENV + let(:test_redis_url) { "redis://redishost:#{redis_port}"} + before(:each) do stub_env(environment_config_file_name, Rails.root.join(config_file_name)) clear_raw_config -- GitLab From fcff73e046ec8dd16dc00b89200cdba5ccd0ca1d Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 15 Jun 2017 06:27:05 +0800 Subject: [PATCH 49/62] implement response to style comments in review discussions --- lib/gitlab/redis/cache.rb | 8 ++++---- lib/gitlab/redis/queues.rb | 2 +- lib/gitlab/redis/shared_state.rb | 2 +- lib/gitlab/redis/wrapper.rb | 8 ++++---- spec/controllers/metrics_controller_spec.rb | 1 + 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index c7b7ce1be0fb..5959c6b53252 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -19,16 +19,16 @@ def default_url end def config_file_name - # otherwise, if ENV set for this class, use it even if it points to a file does not exist + # if ENV set for this class, use it even if it points to a file does not exist file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] return file_name unless file_name.nil? # otherwise, if config files exists for this class, use it file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_CACHE_URL when config file is absent - file_name = super unless File.file?(file_name) - file_name + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 3726f64de549..2059626d804f 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -20,7 +20,7 @@ def default_url end def config_file_name - # otherwise, if ENV set for this class, use it even if it points to a file does not exist + # if ENV set for this class, use it even if it points to a file does not exist file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] return file_name if file_name diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 5c4f8a9817d4..817b2d98a3c5 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -19,7 +19,7 @@ def default_url end def config_file_name - # otherwise, if ENV set for this class, use it even if it points to a file does not exist + # if ENV set for this class, use it even if it points to a file does not exist file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] return file_name if file_name diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index afb2674ed6bc..d7ae01f319c9 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -49,16 +49,16 @@ def default_url end def config_file_name - # otherwise, if ENV set for wrapper class, use it even if it points to a file does not exist + # if ENV set for wrapper class, use it even if it points to a file does not exist file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] return file_name unless file_name.nil? # otherwise, if config files exists for wrapper class, use it file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) - # this will force use of DEFAULT_REDIS_URL when config file is absent - file_name = nil unless File.file?(file_name) - file_name + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil end end diff --git a/spec/controllers/metrics_controller_spec.rb b/spec/controllers/metrics_controller_spec.rb index f7942166618f..c46dd92b454d 100644 --- a/spec/controllers/metrics_controller_spec.rb +++ b/spec/controllers/metrics_controller_spec.rb @@ -37,6 +37,7 @@ it 'returns Caching ping metrics' do get :index + expect(response.body).to match(/^redis_cache_ping_timeout 0$/) expect(response.body).to match(/^redis_cache_ping_success 1$/) expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/) -- GitLab From 37d8d082ab56a103672222223d5babb2d8d265f2 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 15 Jun 2017 07:31:45 +0800 Subject: [PATCH 50/62] catch up with merged changes from "master" --- config/initializers/peek.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index 65432caac2a6..c700c8956cec 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -1,4 +1,4 @@ -Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) } +Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.Wrapper.params) } Peek.into Peek::Views::Host Peek.into Peek::Views::PerformanceBar -- GitLab From f097e84666cf53a693b3ac103338a27df9c883d4 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 15 Jun 2017 08:06:11 +0800 Subject: [PATCH 51/62] catch up with merged changes from "master" --- config/initializers/peek.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index c700c8956cec..fa84051257d0 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -1,4 +1,4 @@ -Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.Wrapper.params) } +Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Wrapper.params) } Peek.into Peek::Views::Host Peek.into Peek::Views::PerformanceBar -- GitLab From 201572e6d47e0cdb46d264fe374f2bbacc6689d6 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 15 Jun 2017 12:49:32 +0800 Subject: [PATCH 52/62] catch up with merged changes from "master" --- config/initializers/peek.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index fa84051257d0..da8282ec9243 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -1,4 +1,4 @@ -Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Wrapper.params) } +Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) } Peek.into Peek::Views::Host Peek.into Peek::Views::PerformanceBar -- GitLab From a5c63a879942eb0a88593c15b2b7116c5edc6dba Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 16 Jun 2017 06:48:29 +0800 Subject: [PATCH 53/62] implement response to comments in review discussions --- lib/gitlab/redis/cache.rb | 6 ++---- lib/gitlab/redis/queues.rb | 6 ++---- lib/gitlab/redis/shared_state.rb | 4 ++-- lib/gitlab/redis/wrapper.rb | 6 ++---- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 5959c6b53252..d784cb0cab00 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -7,13 +7,11 @@ class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - unless Rails.root.nil? - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.cache.yml').freeze + if defined?(::Rails) && !::Rails.root.nil? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze end class << self - delegate :params, :url, to: :new - def default_url DEFAULT_REDIS_CACHE_URL end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 2059626d804f..055e08141d33 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,13 +8,11 @@ class Queues < ::Gitlab::Redis::Wrapper MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - unless Rails.root.nil? - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.queues.yml').freeze + if defined?(::Rails) && !::Rails.root.nil? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze end class << self - delegate :params, :url, to: :new - def default_url DEFAULT_REDIS_QUEUES_URL end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 817b2d98a3c5..329a9ef2b3b5 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,8 +7,8 @@ class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - unless Rails.root.nil? - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = Rails.root.join('config', 'redis.shared_state.yml').freeze + if defined?(::Rails) && !::Rails.root.nil? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze end class << self diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index d7ae01f319c9..278ed9cc516d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -1,7 +1,5 @@ # This file should only be used by sub-classes, not directly by any clients of the sub-classes # please require all dependencies below: -require 'rails' -require 'redis' require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' @@ -10,8 +8,8 @@ module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - unless Rails.root.nil? - DEFAULT_REDIS_CONFIG_FILE_NAME = Rails.root.join('config', 'resque.yml').freeze + if defined?(::Rails) && !::Rails.root.nil? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze end class << self -- GitLab From cd0a5d97d8eb52c0c07cbb80d6bade88755c5600 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 16 Jun 2017 07:03:17 +0800 Subject: [PATCH 54/62] implement response to comments in review discussions --- scripts/prepare_build.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 25aa977db305..68114d149c4c 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -39,21 +39,12 @@ sed -i 's/localhost/redis/g' config/resque.yml cp config/redis.cache.yml.example config/redis.cache.yml sed -i 's/localhost/redis/g' config/redis.cache.yml -# -# notes for when CI supports service configuration on shared image -#sed -i 's/localhost/caching/g' config/redis.cache.yml cp config/redis.queues.yml.example config/redis.queues.yml sed -i 's/localhost/redis/g' config/redis.queues.yml -# -# notes for when CI supports service configuration on shared image -#sed -i 's/localhost/queues/g' config/redis.queues.yml cp config/redis.shared_state.yml.example config/redis.shared_state.yml sed -i 's/localhost/redis/g' config/redis.shared_state.yml -# -# notes for when CI supports service configuration on shared image -#sed -i 's/localhost/redis_shared_state/g' config/redis.shared_state.yml if [ "$SETUP_DB" != "false" ]; then bundle exec rake db:drop db:create db:schema:load db:migrate -- GitLab From 969ac7b56c629f43080d9f17967b3fa147cbb24d Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Wed, 21 Jun 2017 10:57:21 +0800 Subject: [PATCH 55/62] updated per MR review comments --- lib/gitlab/redis/cache.rb | 2 +- lib/gitlab/redis/queues.rb | 2 +- lib/gitlab/redis/shared_state.rb | 4 +--- lib/gitlab/redis/wrapper.rb | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index d784cb0cab00..b0da516ff83e 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -7,7 +7,7 @@ class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - if defined?(::Rails) && !::Rails.root.nil? + if defined?(::Rails) && ::Rails.root.present? DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 055e08141d33..f9249d055653 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,7 +8,7 @@ class Queues < ::Gitlab::Redis::Wrapper MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - if defined?(::Rails) && !::Rails.root.nil? + if defined?(::Rails) && ::Rails.root.present? DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 329a9ef2b3b5..395dcf082da2 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,13 +7,11 @@ class SharedState < ::Gitlab::Redis::Wrapper SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - if defined?(::Rails) && !::Rails.root.nil? + if defined?(::Rails) && ::Rails.root.present? DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze end class << self - delegate :params, :url, to: :new - def default_url DEFAULT_REDIS_SHARED_STATE_URL end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 278ed9cc516d..10c16a962ca1 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,7 +8,7 @@ module Redis class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - if defined?(::Rails) && !::Rails.root.nil? + if defined?(::Rails) && ::Rails.root.present? DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze end -- GitLab From 254ad7b9bf1ddbaf7fc957a2795cfaa010254c83 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 29 Jun 2017 17:59:47 +0700 Subject: [PATCH 56/62] update to sync changes in upstream --- lib/gitlab/exclusive_lease.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 7e6bc23c686a..3784f6c4947e 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -53,8 +53,8 @@ def try_obtain # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end -- GitLab From 108c48d3858b2244b75732299a7977ba266732d8 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Thu, 29 Jun 2017 19:49:30 +0700 Subject: [PATCH 57/62] update incorrect auto-merge of spec file --- .../migrate_user_activities_to_users_last_activity_on_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index a7be02f24278..063829be5468 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') -describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state do +describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state, :truncate do let(:migration) { described_class.new } let!(:user_active_1) { create(:user) } let!(:user_active_2) { create(:user) } -- GitLab From 2dc57ed154186c74db63469e3756fb92785ea284 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 7 Jul 2017 07:57:05 +0700 Subject: [PATCH 58/62] catch up with other changes in the code base --- .../rename_reserved_paths_migration/v1/rename_base.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61f..1a697396ff1f 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ def remove_cached_html_for_projects(project_ids) def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ def track_rename(type, old_path, new_path) def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) -- GitLab From f6a2f24d80239f4b249775db30ce46866a51329b Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 7 Jul 2017 07:59:58 +0700 Subject: [PATCH 59/62] catch up with other changes in the code base --- .../rename_reserved_paths_migration/v1/rename_base_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 8813f129ef5d..051fde218873 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -236,7 +236,7 @@ def migration_project(project) subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed') old_path, new_path = [nil, nil] - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| rename_info = redis.lpop(key) old_path, new_path = JSON.parse(rename_info) end @@ -268,7 +268,7 @@ def migration_project(project) key = 'rename:FakeRenameReservedPathMigrationV1:project' stored_renames = nil rename_count = 0 - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedStata.with do |redis| stored_renames = redis.lrange(key, 0, 1) rename_count = redis.llen(key) end -- GitLab From ed6283d474a9e488f4cdf444cd312d279aa0ed05 Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Fri, 7 Jul 2017 09:35:15 +0700 Subject: [PATCH 60/62] catch up with other changes in the code base fixed stray typo --- .../rename_reserved_paths_migration/v1/rename_base_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 051fde218873..df7d1b5d27a0 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -268,7 +268,7 @@ def migration_project(project) key = 'rename:FakeRenameReservedPathMigrationV1:project' stored_renames = nil rename_count = 0 - Gitlab::Redis::SharedStata.with do |redis| + Gitlab::Redis::SharedState.with do |redis| stored_renames = redis.lrange(key, 0, 1) rename_count = redis.llen(key) end -- GitLab From b80e47e1bff880d56786635fae22dbb6d1131b7e Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 11 Jul 2017 06:56:36 +0800 Subject: [PATCH 61/62] catch up with other changes in the code base --- spec/lib/gitlab/performance_bar_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/performance_bar_spec.rb b/spec/lib/gitlab/performance_bar_spec.rb index 8a586bdbf630..b8a2267f1a4f 100644 --- a/spec/lib/gitlab/performance_bar_spec.rb +++ b/spec/lib/gitlab/performance_bar_spec.rb @@ -7,7 +7,7 @@ described_class.enabled?(user) end - it 'caches the allowed user IDs in cache', :caching do + it 'caches the allowed user IDs in cache', :use_clean_rails_memory_store_caching do expect do expect(described_class.enabled?(user)).to be_truthy end.not_to exceed_query_limit(0) -- GitLab From 060422bbfeefe61c3a06a63c42a14a010fe729db Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 11 Jul 2017 07:35:19 +0800 Subject: [PATCH 62/62] restore changes needed for multi-redis # Conflicts: (resolved) # config/initializers/sidekiq.rb --- config/initializers/sidekiq.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 3be4cd797aa4..a1cc9655319c 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,12 +1,12 @@ -# Custom Redis configuration -redis_config_hash = Gitlab::Redis.params -redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE +# Custom Queues configuration +queues_config_hash = Gitlab::Redis::Queues.params +queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } Sidekiq.configure_server do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] @@ -54,7 +54,7 @@ end Sidekiq.configure_client do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.client_middleware do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware -- GitLab